Merge pull request #3821 from fluffyfreak/threading-modelcompiler

Multi-Threading modelcompiler

src/Makefile.am

@@ -428,6 +428,7 @@ modelcompiler_SOURCES = \
 	FontCache.cpp \
 	IniConfig.cpp \
 	GameConfig.cpp \
+	JobQueue.cpp \
 	Lang.cpp \
 	ModManager.cpp \
 	NavLights.cpp \

src/graphics/TextureBuilder.cpp

@@ -12,6 +12,9 @@
 
 namespace Graphics {
 
+//static 
+SDL_mutex *TextureBuilder::m_textureLock = nullptr;
+
 TextureBuilder::TextureBuilder(const SDLSurfacePtr &surface, TextureSampleMode sampleMode, bool generateMipmaps, bool potExtend, bool forceRGBA, bool compressTextures, bool anisoFiltering) :
     m_surface(surface), m_sampleMode(sampleMode), m_generateMipmaps(generateMipmaps), m_potExtend(potExtend), m_forceRGBA(forceRGBA), m_compressTextures(compressTextures), m_anisotropicFiltering(anisoFiltering), m_textureType(TEXTURE_2D), m_prepared(false)
 {
@@ -26,6 +29,11 @@ TextureBuilder::~TextureBuilder()
 {
 }
 
+void TextureBuilder::Init()
+{
+	m_textureLock = SDL_CreateMutex();
+}
+
 // RGBA and RGBpixel format for converting textures
 // XXX little-endian. if we ever have a port to a big-endian arch, invert shift and mask
 #if SDL_BYTEORDER != SDL_LIL_ENDIAN

src/graphics/TextureBuilder.h

@@ -20,6 +20,8 @@ class TextureBuilder {
 	TextureBuilder(const std::string &filename, TextureSampleMode sampleMode = LINEAR_CLAMP, bool generateMipmaps = false, bool potExtend = false, bool forceRGBA = true, bool compressTextures = true, bool anisoFiltering = true, TextureType textureType = TEXTURE_2D);
 	~TextureBuilder();
 
+	static void Init();
+
 	// convenience constructors for common texture types
 	static TextureBuilder Model(const std::string &filename) {
 		return TextureBuilder(filename, LINEAR_REPEAT, true, false, false, true, true);
@@ -49,10 +51,12 @@ class TextureBuilder {
 		if(m_filename.empty()) {
 			return CreateTexture(r);
 		}
+		SDL_LockMutex(m_textureLock);
 		Texture *t = r->GetCachedTexture(type, m_filename);
-		if (t) return t;
+		if (t) { SDL_UnlockMutex(m_textureLock); return t; }
 		t = CreateTexture(r);
 		r->AddCachedTexture(type, m_filename, t);
+		SDL_UnlockMutex(m_textureLock);
 		return t;
 	}
 
@@ -88,6 +92,8 @@ class TextureBuilder {
 
 	void LoadSurface();
 	void LoadDDS();
+
+	static SDL_mutex *m_textureLock;
 };
 
 }

src/graphics/opengl/RendererGL.cpp

@@ -8,6 +8,7 @@
 #include "OS.h"
 #include "StringF.h"
 #include "graphics/Texture.h"
+#include "graphics/TextureBuilder.h"
 #include "TextureGL.h"
 #include "graphics/VertexArray.h"
 #include "GLDebug.h"
@@ -85,6 +86,8 @@ RendererOGL::RendererOGL(WindowSDL *window, const Graphics::Settings &vs)
 			);
 	}
 
+	TextureBuilder::Init();
+
 	m_viewportStack.push(Viewport());
 
 	const bool useDXTnTextures = vs.useTextureCompression;

src/modelcompiler.cpp

@@ -10,6 +10,7 @@
 
 #include "FileSystem.h"
 #include "GameConfig.h"
+#include "JobQueue.h"
 #include "graphics/dummy/RendererDummy.h"
 #include "graphics/Graphics.h"
 #include "graphics/Light.h"
@@ -28,9 +29,36 @@
 
 std::unique_ptr<GameConfig> s_config;
 std::unique_ptr<Graphics::Renderer> s_renderer;
+std::unique_ptr<AsyncJobQueue> asyncJobQueue;
 
 static const std::string s_dummyPath("");
 
+// fwd decl'
+void RunCompiler(const std::string &modelName, const std::string &filepath, const bool bInPlace);
+
+// ********************************************************************************
+// Overloaded PureJob class to handle compiling each model
+// ********************************************************************************
+class CompileJob : public Job
+{
+public:
+	CompileJob() {};
+	CompileJob(const std::string &name, const std::string &path, const bool inPlace)
+		: m_name(name), m_path(path), m_inPlace(inPlace) {}
+
+	virtual void OnRun() override final { RunCompiler(m_name, m_path, m_inPlace); }    // RUNS IN ANOTHER THREAD!! MUST BE THREAD SAFE!
+	virtual void OnFinish() override final {}
+	virtual void OnCancel() override final {}
+
+protected:
+	std::string	m_name;
+	std::string	m_path;
+	bool		m_inPlace;
+};
+
+// ********************************************************************************
+// functions
+// ********************************************************************************
 void SetupRenderer()
 {
 	PROFILE_SCOPED()
@@ -61,6 +89,15 @@ void SetupRenderer()
 	videoSettings.iconFile = OS::GetIconFilename();
 	videoSettings.title = "Model Compiler";
 	s_renderer.reset(Graphics::Init(videoSettings));
+
+	// get threads up
+	Uint32 numThreads = s_config->Int("WorkerThreads");
+	const int numCores = OS::GetNumCores();
+	assert(numCores > 0);
+	if (numThreads == 0) 
+		numThreads = std::max(Uint32(numCores), 1U); // this is a tool, we can use all of the cores for processing unlike Pioneer
+	asyncJobQueue.reset(new AsyncJobQueue(numThreads));
+	Output("started %d worker threads\n", numThreads);
 }
 
 void RunCompiler(const std::string &modelName, const std::string &filepath, const bool bInPlace)
@@ -100,6 +137,9 @@ void RunCompiler(const std::string &modelName, const std::string &filepath, cons
 }
 
 
+// ********************************************************************************
+// functions
+// ********************************************************************************
 enum RunMode {
 	MODE_MODELCOMPILER=0,
 	MODE_MODELBATCHEXPORT,
@@ -221,8 +261,19 @@ int main(int argc, char** argv)
 			}
 
 			SetupRenderer();
+			std::deque<Job::Handle> handles;
 			for (auto &modelName : list_model) {
-				RunCompiler(modelName.first, modelName.second, isInPlace);
+				handles.push_back( asyncJobQueue->Queue(new CompileJob(modelName.first, modelName.second, isInPlace)) );
+			}
+
+			while(true) {
+				asyncJobQueue->FinishJobs();
+				bool hasJobs = false;
+				for(auto &handle : handles) 
+					hasJobs |= handle.HasJob();
+
+				if(!hasJobs)
+					break;
 			}
 			break;
 		}

win32/vs2015/modelcompiler.vcxproj

@@ -27,6 +27,7 @@
     <ClCompile Include="..\..\src\FileSystem.cpp" />
     <ClCompile Include="..\..\src\GameConfig.cpp" />
     <ClCompile Include="..\..\src\IniConfig.cpp" />
+    <ClCompile Include="..\..\src\JobQueue.cpp" />
     <ClCompile Include="..\..\src\Lang.cpp" />
     <ClCompile Include="..\..\src\modelcompiler.cpp" />
     <ClCompile Include="..\..\src\ModManager.cpp" />
@@ -50,6 +51,7 @@
     <ClInclude Include="..\..\src\FileSystem.h" />
     <ClInclude Include="..\..\src\GameConfig.h" />
     <ClInclude Include="..\..\src\IniConfig.h" />
+    <ClInclude Include="..\..\src\JobQueue.h" />
     <ClInclude Include="..\..\src\Lang.h" />
     <ClInclude Include="..\..\src\LangStrings.inc.h" />
     <ClInclude Include="..\..\src\libs.h" />

win32/vs2015/modelcompiler.vcxproj.filters

@@ -78,6 +78,9 @@
     <ClCompile Include="..\..\src\win32\TextUtils.cpp">
       <Filter>Win32</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\JobQueue.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\src\IniConfig.h">
@@ -164,5 +167,8 @@
     <ClInclude Include="..\..\src\DateTime.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\src\JobQueue.h">
+      <Filter>Source Files</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>