diff --git a/src/cgame/cg_main.cpp b/src/cgame/cg_main.cpp index 11de933bab..cf0093b5ff 100644 --- a/src/cgame/cg_main.cpp +++ b/src/cgame/cg_main.cpp @@ -158,6 +158,9 @@ Cvar::Cvar cg_motionblurMinSpeed("cg_motionblurMinSpeed", "minimum speed Cvar::Cvar cg_spawnEffects("cg_spawnEffects", "desaturate world view when dead or spawning", Cvar::NONE, true); static Cvar::Cvar cg_navgenOnLoad("cg_navgenOnLoad", "generate navmeshes when starting a local game", Cvar::NONE, true); +static Cvar::Cvar cg_navgenMaxThreads( + "cg_navgenMaxThreads", "Maximum number of threads to use when generating navmeshes", + Cvar::NONE, 2); // search 'fovCvar' to find usage of these (names come from config files) // 0 means use global FOV setting @@ -1075,7 +1078,7 @@ bool CG_ClientIsReady( int clientNum ) static void GenerateNavmeshes() { std::string mapName = Cvar::GetValue( "mapname" ); - std::vector missing; + std::bitset missing; NavgenConfig config = ReadNavgenConfig( mapName ); bool reduceTypes; if ( !Cvar::ParseCvarValue( Cvar::GetValue( "g_bot_navmeshReduceTypes" ), reduceTypes ) ) @@ -1089,7 +1092,7 @@ static void GenerateNavmeshes() std::string filename = NavmeshFilename( mapName, species ); if ( BG_FOpenGameOrPakPath( filename, f ) < 0 ) { - missing.push_back( species ); + missing.set(species); continue; } NavMeshSetHeader header; @@ -1097,11 +1100,11 @@ static void GenerateNavmeshes() if ( !error.empty() ) { Log::Notice( "Existing navmesh file %s can't be used: %s", filename, error ); - missing.push_back( species ); + missing.set(species); } trap_FS_FCloseFile( f ); } - if ( missing.empty() ) + if ( !missing.any() ) { return; } @@ -1113,30 +1116,16 @@ static void GenerateNavmeshes() trap_UpdateScreen(); NavmeshGenerator navgen; - navgen.Init( mapName ); - float classesCompleted = 0.3; // Assume that Init() is 0.3 times as much work as generating 1 species - // and assume that each species takes the same amount of time, which is actually completely wrong: - // smaller ones take much longer - float classesTotal = classesCompleted + missing.size(); - for ( class_t species : missing ) - { - cg.loadingText = - Str::Format( "%s — %s", message, BG_ClassModelConfig( species )->humanName ); - cg.navmeshLoadingFraction = classesCompleted / classesTotal; - trap_UpdateScreen(); - - std::unique_ptr task = navgen.StartGeneration( species ); - do + navgen.EnqueueTasks( mapName, missing ); + navgen.StartBackgroundThreads( cg_navgenMaxThreads.Get() ); + navgen.WaitInMainThread( []( float progress ) { + if ( progress >= cg.navmeshLoadingFraction + 0.01f ) { - float fraction = ( classesCompleted + task->FractionCompleted() ) / classesTotal; - if ( fraction - cg.navmeshLoadingFraction > 0.01 ) - { - cg.navmeshLoadingFraction = fraction; - trap_UpdateScreen(); - } - } while ( !navgen.Step( *task ) ); - ++classesCompleted; - } + cg.navmeshLoadingFraction = progress; + trap_UpdateScreen(); + } + } ); + cg.loadingNavmesh = false; } diff --git a/src/sgame/sg_bot_nav.cpp b/src/sgame/sg_bot_nav.cpp index 289fc3bf23..dab3fa7a75 100644 --- a/src/sgame/sg_bot_nav.cpp +++ b/src/sgame/sg_bot_nav.cpp @@ -49,28 +49,18 @@ Navigation Mesh Generation =========================== */ +static Cvar::Cvar g_bot_navgen_maxThreads( + "g_bot_navgen_maxThreads", "Number of background threads to use when generating navmeshes. 0 for main thread", + Cvar::NONE, 1); + // blocks the main thread! void G_BlockingGenerateNavmesh( std::bitset classes ) { std::string mapName = Cvar::GetValue( "mapname" ); NavmeshGenerator navgen; - - for ( int i = PCL_NONE; ++i < PCL_NUM_CLASSES; ) - { - if ( !classes[ i ] ) - { - continue; - } - - navgen.Init( mapName ); - std::unique_ptr task = navgen.StartGeneration( Util::enum_cast( i ) ); - - while ( !navgen.Step( *task ) ) - { - // ping the engine with a useless message so that it does not think the sgame VM has hung - Cvar::GetValue( "x" ); - } - } + navgen.EnqueueTasks( mapName, classes ); + navgen.StartBackgroundThreads( g_bot_navgen_maxThreads.Get() ); + navgen.WaitInMainThread( []( float ) {} ); } // TODO: Latch(), when supported in gamelogic @@ -86,44 +76,29 @@ static Cvar::Cvar frameToggle("g_bot_navgen_frame", "FOR INTERNAL USE", Cva static Cvar::Cvar g_bot_autocrouch("g_bot_autocrouch", "whether bots should crouch when they detect an obstacle", Cvar::NONE, true); static NavmeshGenerator navgen; -static std::vector navgenQueue; -static std::unique_ptr generatingNow; +bool usingBackgroundThreads; +static std::unique_ptr generatingNow; // used if generating on the main thread static int nextLogTime; static void G_BotBackgroundNavgenShutdown() { navgen.~NavmeshGenerator(); new (&navgen) NavmeshGenerator(); - navgenQueue.clear(); generatingNow.reset(); } -void G_BotBackgroundNavgen() +// Returns true if done +static bool MainThreadBackgroundNavgen() { - if ( navgenQueue.empty() ) - { - return; - } - - ASSERT_EQ( navMeshLoaded, navMeshStatus_t::GENERATING ); - int stopTime = Sys::Milliseconds() + msecPerFrame.Get(); - navgen.Init( Cvar::GetValue( "mapname" ) ); - if ( !generatingNow ) { - class_t next = navgenQueue.back(); - generatingNow = navgen.StartGeneration( next ); - } - - if ( level.time > nextLogTime ) - { - std::string percent = std::to_string( int(generatingNow->FractionCompleted() * 100) ); - trap_SendServerCommand( -1, va( "print_tr %s %s %s", - QQ( N_( "Server: generating bot navigation mesh for $1$... $2$%" ) ), - BG_Class( generatingNow->species )->name, percent.c_str() ) ); - nextLogTime = level.time + 10000; + generatingNow = navgen.PopTask(); + if ( !generatingNow ) + { + return true; + } } // HACK: if the game simulation time gets behind the real time, the server runs a bunch of @@ -135,7 +110,7 @@ void G_BotBackgroundNavgen() static int lastToggle = -12345; if ( lastToggle == frameToggle.Get() ) { - return; + return false; } lastToggle = frameToggle.Get(); trap_SendConsoleCommand( "toggle g_bot_navgen_frame" ); @@ -144,14 +119,35 @@ void G_BotBackgroundNavgen() { if ( navgen.Step( *generatingNow ) ) { - ASSERT_EQ( generatingNow->species, navgenQueue.back() ); - navgenQueue.pop_back(); + generatingNow->context.DoMainThreadTasks(); generatingNow.reset(); break; } } - if ( navgenQueue.empty() ) // finished? + return false; +} + +void G_BotBackgroundNavgen() +{ + if ( navMeshLoaded != navMeshStatus_t::GENERATING ) + { + return; + } + + bool done; + + if ( usingBackgroundThreads ) + { + done = navgen.ThreadsDone(); + navgen.HandleFinishedTasks(); + } + else + { + done = MainThreadBackgroundNavgen(); + } + + if ( done ) { G_BotBackgroundNavgenShutdown(); @@ -169,6 +165,13 @@ void G_BotBackgroundNavgen() // Bots on spectate will now join in their think function } } + else if ( level.time > nextLogTime ) + { + std::string percent = std::to_string( int(navgen.FractionComplete() * 100) ); + trap_SendServerCommand( -1, va( "print_tr %s %s", + QQ( N_( "Server: generating bot navigation meshes... $1$%" ) ), percent.c_str() ) ); + nextLogTime = level.time + 10000; + } } /* @@ -195,7 +198,8 @@ void G_BotNavInit( int generateNeeded ) } std::bitset missing; - NavgenConfig config = ReadNavgenConfig( Cvar::GetValue( "mapname" ) ); + std::string mapName = Cvar::GetValue( "mapname" ); + NavgenConfig config = ReadNavgenConfig( mapName ); for ( class_t i : RequiredNavmeshes( g_bot_navmeshReduceTypes.Get() ) ) { @@ -231,14 +235,15 @@ void G_BotNavInit( int generateNeeded ) } else { - ASSERT( navgenQueue.empty() ); - for ( int i = PCL_NUM_CLASSES; --i > PCL_NONE; ) + ASSERT( !generatingNow ); + navgen.EnqueueTasks( mapName, missing ); + usingBackgroundThreads = g_bot_navgen_maxThreads.Get() > 0; + + if ( usingBackgroundThreads ) { - if ( missing[ i ] ) - { - navgenQueue.push_back( Util::enum_cast( i ) ); - } + navgen.StartBackgroundThreads( g_bot_navgen_maxThreads.Get() ); } + navMeshLoaded = navMeshStatus_t::GENERATING; return; } diff --git a/src/shared/navgen/nav.cpp b/src/shared/navgen/nav.cpp index 7c1aafbde9..ca949be69b 100644 --- a/src/shared/navgen/nav.cpp +++ b/src/shared/navgen/nav.cpp @@ -40,7 +40,8 @@ #include "shared/bg_gameplay.h" #include "navgen.h" -static Log::Logger LOG( VM_STRING_PREFIX "navgen", "", Log::Level::NOTICE ); +// disable suppression in case multiple threads finishing together lead to a burst of output +static auto LOG = Log::Logger( VM_STRING_PREFIX "navgen", "", Log::Level::NOTICE ).WithoutSuppression(); void UnvContext::doLog(rcLogCategory category, const char* msg, int len) { @@ -49,15 +50,28 @@ void UnvContext::doLog(rcLogCategory category, const char* msg, int len) { case RC_LOG_WARNING: case RC_LOG_ERROR: - LOG.Warn( line ); + RunOnMainThread( [line] { LOG.Warn( line ); } ); break; case RC_LOG_PROGRESS: default: - LOG.Notice( line ); + RunOnMainThread( [line] { LOG.Notice( line ); } ); break; } } +void UnvContext::RunOnMainThread( std::function f ) +{ + mainThreadTasks_.push_back( std::move( f ) ); +} + +void UnvContext::DoMainThreadTasks() +{ + for ( auto &f : mainThreadTasks_ ) + { + f(); + } +} + static NavgenStatus::Code CodeForFailedDtStatus(dtStatus status) { if ( dtStatusDetail( status, DT_OUT_OF_MEMORY ) ) @@ -70,6 +84,15 @@ static NavgenStatus::Code CodeForFailedDtStatus(dtStatus status) static int tileSize = 64; void NavmeshGenerator::WriteFile( const NavgenTask &t ) { + if ( t.status.code == NavgenStatus::OK ) + { + LOG.Notice( "Finished generating navmesh for %s", BG_ClassModelConfig( t.species )->humanName ); + } + else + { + LOG.Warn( "Navmesh generation for %s failed: %s", BG_ClassModelConfig( t.species )->humanName, t.status.message ); + } + if ( t.status.code == NavgenStatus::TRANSIENT_FAILURE ) { return; // Don't write anything @@ -1014,10 +1037,43 @@ static NavgenStatus rasterizeTileLayers( Geometry& geo, rcContext &context, int return {}; } +void NavmeshGenerator::EnqueueTasks( Str::StringRef mapName, std::bitset classes ) +{ + // The NavmeshGenerator object is not designed to be used more than once + ASSERT( mapName_.empty() ); + + // never divide by 0 + // This fails to include map loading time but we don't update progress during that anyway + int amountOfWork = 1; + + std::string names; + + for ( int i = PCL_NUM_CLASSES; --i != PCL_NONE; ) + { + if ( !classes[ i ] ) + { + continue; + } + + Init( mapName ); + taskQueue_.push_back( StartGeneration( Util::enum_cast( i ) ) ); + + const NavgenTask& task = *taskQueue_.back(); + if ( task.status.code == NavgenStatus::OK ) + { + amountOfWork += task.tw * task.th; // This correlates pretty well with how long it takes + } + + names = ' ' + ( BG_Class(i)->name + names ); + } + + fractionCompleteDenominator_ = amountOfWork; + LOG.Notice( "Navgen requested for:%s", names ); +} + std::unique_ptr NavmeshGenerator::StartGeneration( class_t species ) { classAttributes_t const& agent = *BG_Class( species ); - LOG.Notice( "Generating navmesh for %s", agent.name ); auto t = Util::make_unique(); t->species = species; t->status = initStatus_; @@ -1059,7 +1115,11 @@ std::unique_ptr NavmeshGenerator::StartGeneration( class_t species ) climb = std::max( config_.stepSize, climb ); } - LOG.Notice( "generating agent %s with stepsize of %d", agent.name, climb ); + // We are actually still running on the main thread, but put the message in there + // so that it will be grouped with other messages about the same class + std::string msg = Str::Format( "generating agent %s with stepsize of %d, grid %dx%d", agent.name, climb, t->tw, t->th ); + t->context.RunOnMainThread( [msg] { LOG.Verbose( msg ); } ); + t->cfg.cs = cellSize; t->cfg.ch = cellHeight_; t->cfg.walkableSlopeAngle = RAD2DEG( acosf( MIN_WALK_NORMAL ) ); @@ -1104,19 +1164,28 @@ std::unique_ptr NavmeshGenerator::StartGeneration( class_t species ) return t; } +std::unique_ptr NavmeshGenerator::PopTask() +{ + if ( taskQueue_.empty() ) + { + return nullptr; + } + + auto ret = std::move( taskQueue_.back() ); + taskQueue_.pop_back(); + return ret; +} + +float NavmeshGenerator::FractionComplete() const +{ + return float(fractionCompleteNumerator_) / float(fractionCompleteDenominator_); +} + bool NavmeshGenerator::Step( NavgenTask &t ) { if ( t.status.code != NavgenStatus::OK || t.y >= t.th || t.tw == 0 ) { - if ( t.status.code == NavgenStatus::OK ) - { - LOG.Verbose( "Finished generating navmesh for %s", BG_ClassModelConfig( t.species )->humanName ); - } - else - { - LOG.Warn( "Navmesh generation for %s failed: %s", BG_ClassModelConfig( t.species )->humanName, t.status.message ); - } - WriteFile( t ); + t.context.RunOnMainThread( [this, &t] { WriteFile( t ); } ); return true; } @@ -1124,7 +1193,7 @@ bool NavmeshGenerator::Step( NavgenTask &t ) memset( tiles, 0, sizeof( tiles ) ); int ntiles; - NavgenStatus status = rasterizeTileLayers(geo_, recastContext_, t.x, t.y, t.cfg, tiles, MAX_LAYERS, !!config_.filterGaps, &ntiles); + NavgenStatus status = rasterizeTileLayers(geo_, t.context, t.x, t.y, t.cfg, tiles, MAX_LAYERS, !!config_.filterGaps, &ntiles); if ( status.code != NavgenStatus::OK ) { t.status = status; @@ -1148,6 +1217,8 @@ bool NavmeshGenerator::Step( NavgenTask &t ) t.x = 0; ++t.y; } + + ++fractionCompleteNumerator_; return false; } @@ -1157,7 +1228,6 @@ void NavmeshGenerator::Init(Str::StringRef mapName) config_ = ReadNavgenConfig( mapName ); mapName_ = mapName; - recastContext_.enableLog(true); initStatus_ = {}; LoadBSP(); LoadGeometry(); @@ -1177,8 +1247,100 @@ void NavmeshGenerator::Init(Str::StringRef mapName) } } -float NavgenTask::FractionCompleted() const +void NavmeshGenerator::StartBackgroundThreads( int numBackgroundThreads ) +{ + numBackgroundThreads = std::max( numBackgroundThreads, 1 ); + numBackgroundThreads = std::min( numBackgroundThreads, static_cast( taskQueue_.size() ) ); + LOG.Notice( "Using %d worker thread(s) for navmesh generation", numBackgroundThreads ); + numActiveThreads_ = numBackgroundThreads; + + for ( ; numBackgroundThreads > 0; numBackgroundThreads-- ) + { + threads_.emplace_back( &NavmeshGenerator::BackgroundThreadMain, this ); + } +} + +void NavmeshGenerator::BackgroundThreadMain() +{ + std::unique_lock lock(taskQueueMutex_); + + while ( true ) + { + std::unique_ptr task; + task = PopTask(); + + if ( !task ) + { + --numActiveThreads_; + return; + } + + lock.unlock(); + int start = Sys::Milliseconds(); + + do + { + if ( canceled_ ) + { + return; + } + } + while ( !Step( *task ) ); + + std::string msg = Str::Format( "Navgen for %s took %d ms", + BG_Class( task->species )->name, Sys::Milliseconds() - start ); + task->context.RunOnMainThread( [msg] { LOG.Verbose( msg ); } ); + lock.lock(); + finishedTasks_.push_back( std::move( task ) ); + } + + --numActiveThreads_; +} + +void NavmeshGenerator::WaitInMainThread( std::function progressCallback ) +{ + while ( !ThreadsDone() ) + { + bool somethingFinished = HandleFinishedTasks(); + progressCallback( FractionComplete() ); + if ( !somethingFinished ) + { + Cvar::GetValue( "x" ); // prevent being killed by engine after 2 seconds of idleness + Sys::SleepFor( std::chrono::milliseconds( 300 ) ); // TODO std::condition_variable? + } + } + + HandleFinishedTasks(); +} + +bool NavmeshGenerator::ThreadsDone() +{ + std::lock_guard lock(taskQueueMutex_); + return numActiveThreads_ == 0; +} + +// returns true if there were any finished +bool NavmeshGenerator::HandleFinishedTasks() { - // Assume that writing the file at the end is 10% - return 0.9f * float(y * tw + x) / float(tw * th); + std::vector> finished; + { + std::lock_guard lock(taskQueueMutex_); + std::swap( finished, finishedTasks_ ); + } + + for ( auto &task : finished ) + { + task->context.DoMainThreadTasks(); + } + + return !finished.empty(); +} + +NavmeshGenerator::~NavmeshGenerator() +{ + canceled_ = true; + for ( std::thread &thread : threads_ ) + { + thread.join(); + } } diff --git a/src/shared/navgen/navgen.h b/src/shared/navgen/navgen.h index 730c3cca1c..34d9d63135 100644 --- a/src/shared/navgen/navgen.h +++ b/src/shared/navgen/navgen.h @@ -20,6 +20,7 @@ =========================================================================== */ +#include #include #include "Recast.h" #include "RecastAlloc.h" @@ -112,7 +113,15 @@ const rcChunkyTriMesh *getChunkyMesh() { return &mesh; } class UnvContext : public rcContext { + std::vector> mainThreadTasks_; + +public: void doLog(rcLogCategory category, const char* msg, int len) override; + + void RunOnMainThread(std::function f); + + // Do this once at the end so that logs from the same class_t are together + void DoMainThreadTasks(); }; struct NavgenStatus @@ -140,14 +149,19 @@ struct NavgenTask { int x = 0; int y = 0; NavgenStatus status; - - float FractionCompleted() const; + UnvContext context; }; + +// There are 3 threading modes of navmesh generation: +// - Main thread does a little bit work once per frame (sgame background generation with threads disabled) +// - Background threads work on navgen while main thread does other things (sgame background with threads enabled) +// - Main thread blocks while background threads run until navgen completed (/navgen and cgame) +// NavgenPool is used for the 2nd and 3rd cases. + // Public interface to navgen. Rest of this file is internal details class NavmeshGenerator { private: - UnvContext recastContext_; NavgenConfig config_; // Custom computed config for the map float cellHeight_; @@ -157,6 +171,11 @@ class NavmeshGenerator { Geometry geo_; NavgenStatus initStatus_; + std::vector> taskQueue_; + + std::atomic fractionCompleteNumerator_{0}; + int fractionCompleteDenominator_ = 0; + // Map geometry loading void LoadBSP(); void LoadGeometry(); @@ -164,13 +183,46 @@ class NavmeshGenerator { void WriteFile(const NavgenTask& t); -public: // load the BSP if it has not been loaded already // in principle mapName could be different from the current map, if the necessary pak is loaded void Init(Str::StringRef mapName); +public: + ~NavmeshGenerator(); + std::unique_ptr StartGeneration(class_t species); + void EnqueueTasks(Str::StringRef mapName, std::bitset classes); + + // Only intended to be meaningful if no tasks have failed + float FractionComplete() const; + + /* + * Optional multi-threading functionality + */ +private: + std::vector threads_; + int numActiveThreads_; + std::atomic canceled_; + std::vector> finishedTasks_; + + // guards taskQueue_, finishedTasks_, numActiveThreads_ during multithreading + std::mutex taskQueueMutex_; + +public: + void StartBackgroundThreads(int numBackgroundThreads); + void WaitInMainThread(std::function progressCallback); + bool ThreadsDone(); + bool HandleFinishedTasks(); + + /* + * METHODS THAT MAY BE CALLED FROM A NON-MAIN THREAD + * Must not use any trap calls, that includes logging!!! + */ +public: + std::unique_ptr PopTask(); // caller must hold mutex if applicable // Returns true when finished bool Step(NavgenTask& t); +private: + void BackgroundThreadMain(); };