diff --git a/source/MaterialXTest/MaterialXRender/RenderUtil.cpp b/source/MaterialXTest/MaterialXRender/RenderUtil.cpp index 2cf720471f..b4d5b7ae06 100644 --- a/source/MaterialXTest/MaterialXRender/RenderUtil.cpp +++ b/source/MaterialXTest/MaterialXRender/RenderUtil.cpp @@ -32,6 +32,26 @@ ShaderRenderTester::~ShaderRenderTester() { } +RenderItem::RenderItem(mx::TypedElementPtr elem, + mx::FileSearchPath searchPath, + mx::FilePath outPath, + mx::ImageVec* images) + : element(std::move(elem)), + imageSearchPath(std::move(searchPath)), + outputPath(std::move(outPath)), + imageVec(images) +{ + static const mx::StringMap pathMap = []() { + mx::StringMap m; + m["/"] = "_"; + m[":"] = "_"; + return m; + }(); + shaderName = mx::createValidName( + mx::replaceSubstrings(element->getNamePath(), pathMap)); + document = element ? element->getDocument() : nullptr; +} + bool ShaderRenderTester::validate(const mx::FilePath optionsFilePath) { // per-run state objects, logger, profiler @@ -57,24 +77,25 @@ bool ShaderRenderTester::validate(const mx::FilePath optionsFilePath) // Data search path runState.searchPath = mx::getDefaultDataSearchPath(); - mx::ScopedTimer ioTimer(&profiler.times().ioTime); - mx::FilePathVec files = collectTestFiles(runState); - ioTimer.endTimer(); + mx::FilePathVec files; + { + mx::ScopedTimer t(&profiler.times().ioTime); + files = collectTestFiles(runState); + } // Load in the library dependencies once. // This will be imported in each test document below. - ioTimer.startTimer(); - loadDependentLibraries(runState); - ioTimer.endTimer(); + { + mx::ScopedTimer t(&profiler.times().ioTime); + loadDependentLibraries(runState); + } // Create renderers and generators initializeGeneratorContext(runState, logger, profiler); - // Map to replace "/" in Element path and ":" in namespaced names with "_". - mx::StringMap pathMap; - pathMap["/"] = "_"; - pathMap[":"] = "_"; + RenderSession session { runState.options, logger.renderLog() }; + bool allSucceeded = true; for (const mx::FilePath& filename : files) { DocumentInfo docInfo = loadAndValidateDocument(filename, runState, logger, profiler); @@ -84,9 +105,12 @@ bool ShaderRenderTester::validate(const mx::FilePath optionsFilePath) for (const auto& element : docInfo.elements) { - mx::string elementName = mx::createValidName(mx::replaceSubstrings(element->getNamePath(), pathMap)); - runRenderer(elementName, element, *runState.context, docInfo.doc, logger.renderLog(), runState.options, - profiler.times(), docInfo.imageSearchPath, docInfo.outputPath, nullptr); + RenderItem item { element, docInfo.imageSearchPath, docInfo.outputPath, nullptr }; + auto result = runRenderer(session, item, *runState.context); + profiler.times().languageTimes.accumulate(result.languageTimes); + profiler.times().elementsTested += result.elementsTested; + if (!result.success) + allSucceeded = false; } } @@ -94,7 +118,7 @@ bool ShaderRenderTester::validate(const mx::FilePath optionsFilePath) // All resource cleanup is handled by destructors. profiler.printSummary(runState.options, logger.profilingLog()); - return true; + return allSucceeded; } void ShaderRenderTester::loadDependentLibraries(TestRunState& runState) @@ -486,11 +510,10 @@ DocumentInfo ShaderRenderTester::loadAndValidateDocument(const mx::FilePath& fil { DocumentInfo info; - mx::ScopedTimer ioTimer(&profiler.times().ioTime); - if (_skipFiles.count(filename.getBaseName()) > 0) return info; + mx::ScopedTimer ioTimer(&profiler.times().ioTime); info.doc = mx::createDocument(); try { @@ -589,13 +612,6 @@ void TestRunLogger::start(const std::string& target, const GenShaderUtil::TestSu #endif } -TestRunLogger::~TestRunLogger() -{ - _renderLog.reset(); - _validationLog.reset(); - _profilingLog.reset(); -} - // --------------------------------------------------------------------------- // TestRunProfiler // --------------------------------------------------------------------------- @@ -646,10 +662,7 @@ void TestRunTracer::start(const std::string& target, const GenShaderUtil::TestSu } } -TestRunTracer::~TestRunTracer() -{ - _state.reset(); -} +TestRunTracer::~TestRunTracer() = default; #endif // MATERIALX_BUILD_PERFETTO_TRACING diff --git a/source/MaterialXTest/MaterialXRender/RenderUtil.h b/source/MaterialXTest/MaterialXRender/RenderUtil.h index 92fb094be6..a19f47ecc4 100644 --- a/source/MaterialXTest/MaterialXRender/RenderUtil.h +++ b/source/MaterialXTest/MaterialXRender/RenderUtil.h @@ -16,6 +16,7 @@ #include #include #include +#include #define LOG_TO_FILE @@ -53,6 +54,19 @@ class LanguageProfileTimes output << "\tI/O: " << ioTime << " seconds" << std::endl; output << "\tImage save: " << imageSaveTime << " seconds" << std::endl; } + + void accumulate(const LanguageProfileTimes& other) + { + totalTime += other.totalTime; + setupTime += other.setupTime; + transparencyTime += other.transparencyTime; + generationTime += other.generationTime; + compileTime += other.compileTime; + renderTime += other.renderTime; + ioTime += other.ioTime; + imageSaveTime += other.imageSaveTime; + } + double totalTime = 0.0; double setupTime = 0.0; double transparencyTime = 0.0; @@ -104,8 +118,6 @@ struct DocumentInfo class TestRunLogger { public: - ~TestRunLogger(); - void start(const std::string& target, const GenShaderUtil::TestSuiteOptions& options); std::ostream& renderLog() { return _renderLog ? *_renderLog : std::cout; } @@ -124,7 +136,7 @@ class TestRunLogger class TestRunProfiler { public: - ~TestRunProfiler() { _totalTimer.reset(); } + ~TestRunProfiler() = default; void start(); void printSummary(const GenShaderUtil::TestSuiteOptions& options, @@ -161,6 +173,41 @@ struct TestRunState std::unique_ptr context; }; +// Read-only test configuration for the entire validate() run. +// Note: the log stream requires synchronization for concurrent use. +struct RenderSession +{ + const GenShaderUtil::TestSuiteOptions& testOptions; + std::ostream& log; +}; + +// Per-element data passed to each runRenderer call. +struct RenderItem +{ + RenderItem(mx::TypedElementPtr elem, + mx::FileSearchPath searchPath, + mx::FilePath outPath, + mx::ImageVec* images = nullptr); + + mx::TypedElementPtr element; + mx::DocumentPtr document; + mx::FileSearchPath imageSearchPath; + mx::FilePath outputPath; + mx::ImageVec* imageVec = nullptr; + std::string shaderName; + + mx::DocumentPtr doc() const { return document; } +}; + +// Returned by runRenderer — each call produces its own isolated profiling data. +// The caller accumulates results, making future parallelism straightforward. +struct RenderProfileResult +{ + LanguageProfileTimes languageTimes; + unsigned int elementsTested = 0; + bool success = true; +}; + // Base class used for performing compilation and render tests for a given // shading language and target. // @@ -209,17 +256,13 @@ class ShaderRenderTester // Create a renderer for the generated code virtual void createRenderer(std::ostream& log) = 0; - // Run the renderer - virtual bool runRenderer(const std::string& shaderName, - mx::TypedElementPtr element, - mx::GenContext& context, - mx::DocumentPtr doc, - std::ostream& log, - const GenShaderUtil::TestSuiteOptions& testOptions, - RenderProfileTimes& profileTimes, - const mx::FileSearchPath& imageSearchPath, - const std::string& outputPath = ".", - mx::ImageVec* imageVec = nullptr) = 0; + // Run the renderer. + // GenContext is a separate argument because it is mutable (written per-element) + // and must be cloned per-thread for future parallel execution. + virtual RenderProfileResult runRenderer( + const RenderSession& session, + const RenderItem& item, + mx::GenContext& context) = 0; // Save an image virtual bool saveImage(const mx::FilePath&, mx::ConstImagePtr, bool) const { return false; }; diff --git a/source/MaterialXTest/MaterialXRenderGlsl/RenderGlsl.cpp b/source/MaterialXTest/MaterialXRenderGlsl/RenderGlsl.cpp index e50810d925..418d9c972b 100644 --- a/source/MaterialXTest/MaterialXRenderGlsl/RenderGlsl.cpp +++ b/source/MaterialXTest/MaterialXRenderGlsl/RenderGlsl.cpp @@ -41,16 +41,10 @@ class GlslShaderRenderTester : public RenderUtil::ShaderRenderTester void createRenderer(std::ostream& log) override; - bool runRenderer(const std::string& shaderName, - mx::TypedElementPtr element, - mx::GenContext& context, - mx::DocumentPtr doc, - std::ostream& log, - const GenShaderUtil::TestSuiteOptions& testOptions, - RenderUtil::RenderProfileTimes& profileTimes, - const mx::FileSearchPath& imageSearchPath, - const std::string& outputPath = ".", - mx::ImageVec* imageVec = nullptr) override; + RenderUtil::RenderProfileResult runRenderer( + const RenderUtil::RenderSession& session, + const RenderUtil::RenderItem& item, + mx::GenContext& context) override; bool saveImage(const mx::FilePath& filePath, mx::ConstImagePtr image, bool verticalFlip) const override; @@ -151,22 +145,23 @@ bool GlslShaderRenderTester::saveImage(const mx::FilePath& filePath, mx::ConstIm return _renderer->getImageHandler()->saveImage(filePath, image, verticalFlip); } -bool GlslShaderRenderTester::runRenderer(const std::string& shaderName, - mx::TypedElementPtr element, - mx::GenContext& context, - mx::DocumentPtr doc, - std::ostream& log, - const GenShaderUtil::TestSuiteOptions& testOptions, - RenderUtil::RenderProfileTimes& profileTimes, - const mx::FileSearchPath& imageSearchPath, - const std::string& outputPath, - mx::ImageVec* imageVec) +RenderUtil::RenderProfileResult GlslShaderRenderTester::runRenderer( + const RenderUtil::RenderSession& session, + const RenderUtil::RenderItem& item, + mx::GenContext& context) { + RenderUtil::RenderProfileResult result; + const std::string& shaderName = item.shaderName; + mx::DocumentPtr doc = item.doc(); + mx::TypedElementPtr element = item.element; + const GenShaderUtil::TestSuiteOptions& testOptions = session.testOptions; + std::ostream& log = session.log; + MX_TRACE_FUNCTION(mx::Tracing::Category::Render); MX_TRACE_SCOPE(mx::Tracing::Category::Material, shaderName.c_str()); std::cout << "Validating GLSL rendering for: " << doc->getSourceUri() << std::endl; - mx::ScopedTimer totalGLSLTime(&profileTimes.languageTimes.totalTime); + mx::ScopedTimer totalGLSLTime(&result.languageTimes.totalTime); const mx::ShaderGenerator& shadergen = context.getShaderGenerator(); mx::FileSearchPath searchPath = mx::getDefaultDataSearchPath(); @@ -180,13 +175,13 @@ bool GlslShaderRenderTester::runRenderer(const std::string& shaderName, for (auto options : optionsList) { - profileTimes.elementsTested++; + result.elementsTested++; - mx::FilePath outputFilePath = outputPath; + mx::FilePath outputFilePath = item.outputPath; // Note: mkdir will fail if the directory already exists which is ok. { - mx::ScopedTimer ioDir(&profileTimes.languageTimes.ioTime); + mx::ScopedTimer ioDir(&result.languageTimes.ioTime); outputFilePath.createDirectory(true); // Use separate directory for reduced output @@ -201,11 +196,11 @@ bool GlslShaderRenderTester::runRenderer(const std::string& shaderName, mx::ShaderPtr shader; try { - mx::ScopedTimer transpTimer(&profileTimes.languageTimes.transparencyTime); + mx::ScopedTimer transpTimer(&result.languageTimes.transparencyTime); options.hwTransparency = mx::isTransparentSurface(element, shadergen.getTarget()); transpTimer.endTimer(); - mx::ScopedTimer generationTimer(&profileTimes.languageTimes.generationTime); + mx::ScopedTimer generationTimer(&result.languageTimes.generationTime); MX_TRACE_SCOPE(mx::Tracing::Category::ShaderGen, "GenerateShader"); mx::GenOptions& contextOptions = context.getOptions(); contextOptions = options; @@ -223,7 +218,8 @@ bool GlslShaderRenderTester::runRenderer(const std::string& shaderName, if (shader == nullptr) { log << ">> Failed to generate shader\n"; - return false; + result.success = false; + return result; } const std::string& vertexSourceCode = shader->getSourceCode(mx::Stage::VERTEX); const std::string& pixelSourceCode = shader->getSourceCode(mx::Stage::PIXEL); @@ -233,7 +229,7 @@ bool GlslShaderRenderTester::runRenderer(const std::string& shaderName, if (testOptions.dumpGeneratedCode) { MX_TRACE_SCOPE(mx::Tracing::Category::Render, "DumpGeneratedCode"); - mx::ScopedTimer dumpTimer(&profileTimes.languageTimes.ioTime); + mx::ScopedTimer dumpTimer(&result.languageTimes.ioTime); std::ofstream file; file.open(shaderPath + "_vs.glsl"); file << vertexSourceCode; @@ -285,7 +281,7 @@ bool GlslShaderRenderTester::runRenderer(const std::string& shaderName, { MX_TRACE_SCOPE(mx::Tracing::Category::Render, "CompileShader"); - mx::ScopedTimer compileTimer(&profileTimes.languageTimes.compileTime); + mx::ScopedTimer compileTimer(&result.languageTimes.compileTime); _renderer->createProgram(shader); _renderer->validateInputs(); } @@ -293,7 +289,7 @@ bool GlslShaderRenderTester::runRenderer(const std::string& shaderName, if (testOptions.dumpUniformsAndAttributes) { MX_TRACE_SCOPE(mx::Tracing::Category::Render, "DumpUniformsAndAttributes"); - mx::ScopedTimer printTimer(&profileTimes.languageTimes.ioTime); + mx::ScopedTimer printTimer(&result.languageTimes.ioTime); log << "* Uniform:" << std::endl; program->printUniforms(log); log << "* Attributes:" << std::endl; @@ -353,8 +349,8 @@ bool GlslShaderRenderTester::runRenderer(const std::string& shaderName, { MX_TRACE_SCOPE(mx::Tracing::Category::Render, "RenderMaterial"); - mx::ScopedTimer renderTimer(&profileTimes.languageTimes.renderTime); - _renderer->getImageHandler()->setSearchPath(imageSearchPath); + mx::ScopedTimer renderTimer(&result.languageTimes.renderTime); + _renderer->getImageHandler()->setSearchPath(item.imageSearchPath); unsigned int width = (unsigned int) testOptions.renderSize[0] * supersampleFactor; unsigned int height = (unsigned int) testOptions.renderSize[1] * supersampleFactor; _renderer->setSize(width, height); @@ -364,7 +360,7 @@ bool GlslShaderRenderTester::runRenderer(const std::string& shaderName, { MX_TRACE_SCOPE(mx::Tracing::Category::Render, "CaptureAndSaveImage"); - mx::ScopedTimer ioTimer(&profileTimes.languageTimes.imageSaveTime); + mx::ScopedTimer ioTimer(&result.languageTimes.imageSaveTime); std::string fileName = shaderPath + "_glsl.png"; mx::ImagePtr image = _renderer->captureImage(); if (image) @@ -374,9 +370,9 @@ bool GlslShaderRenderTester::runRenderer(const std::string& shaderName, image = image->applyBoxDownsample(supersampleFactor); } _renderer->getImageHandler()->saveImage(fileName, image, true); - if (imageVec) + if (item.imageVec) { - imageVec->push_back(image); + item.imageVec->push_back(image); } } } @@ -407,9 +403,11 @@ bool GlslShaderRenderTester::runRenderer(const std::string& shaderName, WARN(std::string(e.what()) + " in " + shaderPath); } CHECK(validated); + if (!validated) + result.success = false; } } - return true; + return result; } TEST_CASE("Render: GLSL TestSuite", "[renderglsl]") diff --git a/source/MaterialXTest/MaterialXRenderMdl/RenderMdl.cpp b/source/MaterialXTest/MaterialXRenderMdl/RenderMdl.cpp index cba33d9c47..8f27729675 100644 --- a/source/MaterialXTest/MaterialXRenderMdl/RenderMdl.cpp +++ b/source/MaterialXTest/MaterialXRenderMdl/RenderMdl.cpp @@ -31,35 +31,30 @@ class MdlShaderRenderTester : public RenderUtil::ShaderRenderTester protected: void createRenderer(std::ostream& /*log*/) override { }; - bool runRenderer(const std::string& shaderName, - mx::TypedElementPtr element, - mx::GenContext& context, - mx::DocumentPtr doc, - std::ostream& log, - const GenShaderUtil::TestSuiteOptions& testOptions, - RenderUtil::RenderProfileTimes& profileTimes, - const mx::FileSearchPath& imageSearchPath, - const std::string& outputPath = ".", - mx::ImageVec* imageVec = nullptr) override; + RenderUtil::RenderProfileResult runRenderer( + const RenderUtil::RenderSession& session, + const RenderUtil::RenderItem& item, + mx::GenContext& context) override; mx::DocumentPtr _last_doc; }; // Renderer execution -bool MdlShaderRenderTester::runRenderer(const std::string& shaderName, - mx::TypedElementPtr element, - mx::GenContext& context, - mx::DocumentPtr doc, - std::ostream& log, - const GenShaderUtil::TestSuiteOptions& testOptions, - RenderUtil::RenderProfileTimes& profileTimes, - const mx::FileSearchPath&, - const std::string& outputPath, - mx::ImageVec* /*imageVec*/) +RenderUtil::RenderProfileResult MdlShaderRenderTester::runRenderer( + const RenderUtil::RenderSession& session, + const RenderUtil::RenderItem& item, + mx::GenContext& context) { + RenderUtil::RenderProfileResult result; + const std::string& shaderName = item.shaderName; + mx::DocumentPtr doc = item.doc(); + mx::TypedElementPtr element = item.element; + const GenShaderUtil::TestSuiteOptions& testOptions = session.testOptions; + std::ostream& log = session.log; + std::cout << "Validating MDL rendering for: " << doc->getSourceUri() << ", element: " << element->getNamePath() << std::endl; - mx::ScopedTimer totalMDLTime(&profileTimes.languageTimes.totalTime); + mx::ScopedTimer totalMDLTime(&result.languageTimes.totalTime); mx::ShaderGenerator& shadergen = context.getShaderGenerator(); @@ -81,12 +76,12 @@ bool MdlShaderRenderTester::runRenderer(const std::string& shaderName, for (const auto& options : optionsList) { - profileTimes.elementsTested++; + result.elementsTested++; mx::ShaderPtr shader; try { - mx::ScopedTimer genTimer(&profileTimes.languageTimes.generationTime); + mx::ScopedTimer genTimer(&result.languageTimes.generationTime); mx::GenOptions& contextOptions = context.getOptions(); contextOptions = options; contextOptions.targetColorSpaceOverride = "lin_rec709"; @@ -108,14 +103,15 @@ bool MdlShaderRenderTester::runRenderer(const std::string& shaderName, if (shader == nullptr) { log << ">> Failed to generate shader\n"; - return false; + result.success = false; + return result; } CHECK(shader->getSourceCode().length() > 0); std::string mdlCmdStr = shader->getSourceCode(); std::string shaderPath; - mx::FilePath outputFilePath = outputPath; + mx::FilePath outputFilePath = item.outputPath; // Use separate directory for reduced output if (options.shaderInterfaceType == mx::SHADER_INTERFACE_REDUCED) { @@ -124,7 +120,7 @@ bool MdlShaderRenderTester::runRenderer(const std::string& shaderName, // Note: mkdir will fail if the directory already exists which is ok. { - mx::ScopedTimer ioDir(&profileTimes.languageTimes.ioTime); + mx::ScopedTimer ioDir(&result.languageTimes.ioTime); outputFilePath.createDirectory(true); } @@ -132,7 +128,7 @@ bool MdlShaderRenderTester::runRenderer(const std::string& shaderName, // Write out mdl file { - mx::ScopedTimer ioTimer(&profileTimes.languageTimes.ioTime); + mx::ScopedTimer ioTimer(&result.languageTimes.ioTime); std::ofstream file; file.open(shaderPath + ".rendermdl.mdl"); file << shader->getSourceCode(); @@ -157,8 +153,8 @@ bool MdlShaderRenderTester::runRenderer(const std::string& shaderName, command += " --mdl_path \"" + sp.asString() + "\""; } - // Set MDL search path for the module itself. - command += " --mdl_path \"" + outputPath + "\""; + // Set MDL search path for the module itself (directory containing the .mdl file). + command += " --mdl_path \"" + outputFilePath.asString() + "\""; // Set environment mx::FileSearchPath searchPath = mx::getDefaultDataSearchPath(); @@ -230,10 +226,12 @@ bool MdlShaderRenderTester::runRenderer(const std::string& shaderName, log << e.what() << "\n"; } CHECK(validated); + if (!validated) + result.success = false; } } - return true; + return result; } TEST_CASE("Render: MDL TestSuite", "[rendermdl]") diff --git a/source/MaterialXTest/MaterialXRenderMsl/RenderMsl.mm b/source/MaterialXTest/MaterialXRenderMsl/RenderMsl.mm index 4a7f5d470a..cbbd5ccac5 100644 --- a/source/MaterialXTest/MaterialXRenderMsl/RenderMsl.mm +++ b/source/MaterialXTest/MaterialXRenderMsl/RenderMsl.mm @@ -44,16 +44,10 @@ void registerLights(mx::DocumentPtr document, const GenShaderUtil::TestSuiteOpti void createRenderer(std::ostream& log) override; - bool runRenderer(const std::string& shaderName, - mx::TypedElementPtr element, - mx::GenContext& context, - mx::DocumentPtr doc, - std::ostream& log, - const GenShaderUtil::TestSuiteOptions& testOptions, - RenderUtil::RenderProfileTimes& profileTimes, - const mx::FileSearchPath& imageSearchPath, - const std::string& outputPath = ".", - mx::ImageVec* imageVec = nullptr) override; + RenderUtil::RenderProfileResult runRenderer( + const RenderUtil::RenderSession& session, + const RenderUtil::RenderItem& item, + mx::GenContext& context) override; bool saveImage(const mx::FilePath& filePath, mx::ConstImagePtr image, bool verticalFlip) const override; @@ -158,20 +152,21 @@ bool runRenderer(const std::string& shaderName, return _renderer->getImageHandler()->saveImage(filePath, image, verticalFlip); } -bool MslShaderRenderTester::runRenderer(const std::string& shaderName, - mx::TypedElementPtr element, - mx::GenContext& context, - mx::DocumentPtr doc, - std::ostream& log, - const GenShaderUtil::TestSuiteOptions& testOptions, - RenderUtil::RenderProfileTimes& profileTimes, - const mx::FileSearchPath& imageSearchPath, - const std::string& outputPath, - mx::ImageVec* imageVec) +RenderUtil::RenderProfileResult MslShaderRenderTester::runRenderer( + const RenderUtil::RenderSession& session, + const RenderUtil::RenderItem& item, + mx::GenContext& context) { + RenderUtil::RenderProfileResult result; + const std::string& shaderName = item.shaderName; + mx::DocumentPtr doc = item.doc(); + mx::TypedElementPtr element = item.element; + const GenShaderUtil::TestSuiteOptions& testOptions = session.testOptions; + std::ostream& log = session.log; + std::cout << "Validating MSL rendering for: " << doc->getSourceUri() << std::endl; - mx::ScopedTimer totalMSLTime(&profileTimes.languageTimes.totalTime); + mx::ScopedTimer totalMSLTime(&result.languageTimes.totalTime); mx::FileSearchPath searchPath = mx::getDefaultDataSearchPath(); const mx::ShaderGenerator& shadergen = context.getShaderGenerator(); @@ -185,13 +180,13 @@ bool runRenderer(const std::string& shaderName, for (auto options : optionsList) { - profileTimes.elementsTested++; + result.elementsTested++; - mx::FilePath outputFilePath = outputPath; + mx::FilePath outputFilePath = item.outputPath; // Note: mkdir will fail if the directory already exists which is ok. { - mx::ScopedTimer ioDir(&profileTimes.languageTimes.ioTime); + mx::ScopedTimer ioDir(&result.languageTimes.ioTime); outputFilePath.createDirectory(true); // Use separate directory for reduced output @@ -206,11 +201,11 @@ bool runRenderer(const std::string& shaderName, mx::ShaderPtr shader; try { - mx::ScopedTimer transpTimer(&profileTimes.languageTimes.transparencyTime); + mx::ScopedTimer transpTimer(&result.languageTimes.transparencyTime); options.hwTransparency = mx::isTransparentSurface(element, shadergen.getTarget()); transpTimer.endTimer(); - mx::ScopedTimer generationTimer(&profileTimes.languageTimes.generationTime); + mx::ScopedTimer generationTimer(&result.languageTimes.generationTime); mx::GenOptions& contextOptions = context.getOptions(); contextOptions = options; contextOptions.targetColorSpaceOverride = "lin_rec709"; @@ -227,7 +222,8 @@ bool runRenderer(const std::string& shaderName, if (shader == nullptr) { log << ">> Failed to generate shader\n"; - return false; + result.success = false; + return result; } const std::string& vertexSourceCode = shader->getSourceCode(mx::Stage::VERTEX); const std::string& pixelSourceCode = shader->getSourceCode(mx::Stage::PIXEL); @@ -236,7 +232,7 @@ bool runRenderer(const std::string& shaderName, if (testOptions.dumpGeneratedCode) { - mx::ScopedTimer dumpTimer(&profileTimes.languageTimes.ioTime); + mx::ScopedTimer dumpTimer(&result.languageTimes.ioTime); std::ofstream file; file.open(shaderPath + "_vs.metal"); file << vertexSourceCode; @@ -285,7 +281,7 @@ bool runRenderer(const std::string& shaderName, bool isShader = mx::elementRequiresShading(element); _renderer->setLightHandler(isShader ? _lightHandler : nullptr); { - mx::ScopedTimer compileTimer(&profileTimes.languageTimes.compileTime); + mx::ScopedTimer compileTimer(&result.languageTimes.compileTime); _renderer->createProgram(shader); _renderer->validateInputs(); } @@ -293,7 +289,7 @@ bool runRenderer(const std::string& shaderName, if (testOptions.dumpUniformsAndAttributes) { MaterialX::MslProgramPtr program = _renderer->getProgram(); - mx::ScopedTimer printTimer(&profileTimes.languageTimes.ioTime); + mx::ScopedTimer printTimer(&result.languageTimes.ioTime); log << "* Uniform:" << std::endl; program->printUniforms(log); log << "* Attributes:" << std::endl; @@ -352,8 +348,8 @@ bool runRenderer(const std::string& shaderName, int supersampleFactor = testOptions.enableReferenceQuality ? 8 : 1; { - mx::ScopedTimer renderTimer(&profileTimes.languageTimes.renderTime); - _renderer->getImageHandler()->setSearchPath(imageSearchPath); + mx::ScopedTimer renderTimer(&result.languageTimes.renderTime); + _renderer->getImageHandler()->setSearchPath(item.imageSearchPath); unsigned int width = (unsigned int) testOptions.renderSize[0] * supersampleFactor; unsigned int height = (unsigned int) testOptions.renderSize[1] * supersampleFactor; _renderer->setSize(width, height); @@ -361,7 +357,7 @@ bool runRenderer(const std::string& shaderName, } { - mx::ScopedTimer ioTimer(&profileTimes.languageTimes.imageSaveTime); + mx::ScopedTimer ioTimer(&result.languageTimes.imageSaveTime); std::string fileName = shaderPath + "_msl.png"; mx::ImagePtr image = _renderer->captureImage(); if (image) @@ -371,9 +367,9 @@ bool runRenderer(const std::string& shaderName, image = image->applyBoxDownsample(supersampleFactor); } _renderer->getImageHandler()->saveImage(fileName, image, true); - if (imageVec) + if (item.imageVec) { - imageVec->push_back(image); + item.imageVec->push_back(image); } } } @@ -404,9 +400,11 @@ bool runRenderer(const std::string& shaderName, WARN(std::string(e.what()) + " in " + shaderPath); } CHECK(validated); + if (!validated) + result.success = false; } } - return true; + return result; } TEST_CASE("Render: MSL TestSuite", "[rendermsl]") diff --git a/source/MaterialXTest/MaterialXRenderOsl/RenderOsl.cpp b/source/MaterialXTest/MaterialXRenderOsl/RenderOsl.cpp index ab84f8076f..9adf3b335a 100644 --- a/source/MaterialXTest/MaterialXRenderOsl/RenderOsl.cpp +++ b/source/MaterialXTest/MaterialXRenderOsl/RenderOsl.cpp @@ -92,16 +92,10 @@ class OslShaderRenderTester : public RenderUtil::ShaderRenderTester protected: void createRenderer(std::ostream& log) override; - bool runRenderer(const std::string& shaderName, - mx::TypedElementPtr element, - mx::GenContext& context, - mx::DocumentPtr doc, - std::ostream& log, - const GenShaderUtil::TestSuiteOptions& testOptions, - RenderUtil::RenderProfileTimes& profileTimes, - const mx::FileSearchPath& imageSearchPath, - const std::string& outputPath = ".", - mx::ImageVec* imageVec = nullptr) override; + RenderUtil::RenderProfileResult runRenderer( + const RenderUtil::RenderSession& session, + const RenderUtil::RenderItem& item, + mx::GenContext& context) override; void addSkipFiles() override { @@ -194,20 +188,21 @@ void OslShaderRenderTester::createRenderer(std::ostream& log) } // Renderer execution -bool OslShaderRenderTester::runRenderer(const std::string& shaderName, - mx::TypedElementPtr element, - mx::GenContext& context, - mx::DocumentPtr doc, - std::ostream& log, - const GenShaderUtil::TestSuiteOptions& testOptions, - RenderUtil::RenderProfileTimes& profileTimes, - const mx::FileSearchPath&, - const std::string& outputPath, - mx::ImageVec* imageVec) +RenderUtil::RenderProfileResult OslShaderRenderTester::runRenderer( + const RenderUtil::RenderSession& session, + const RenderUtil::RenderItem& item, + mx::GenContext& context) { + RenderUtil::RenderProfileResult result; + const std::string& shaderName = item.shaderName; + mx::DocumentPtr doc = item.doc(); + mx::TypedElementPtr element = item.element; + const GenShaderUtil::TestSuiteOptions& testOptions = session.testOptions; + std::ostream& log = session.log; + std::cout << "Validating OSL rendering for: " << doc->getSourceUri() << std::endl; - mx::ScopedTimer totalOSLTime(&profileTimes.languageTimes.totalTime); + mx::ScopedTimer totalOSLTime(&result.languageTimes.totalTime); mx::ShaderGenerator& shadergen = context.getShaderGenerator(); @@ -220,12 +215,12 @@ bool OslShaderRenderTester::runRenderer(const std::string& shaderName, for (const auto& options : optionsList) { - profileTimes.elementsTested++; + result.elementsTested++; mx::ShaderPtr shader; try { - mx::ScopedTimer genTimer(&profileTimes.languageTimes.generationTime); + mx::ScopedTimer genTimer(&result.languageTimes.generationTime); mx::GenOptions& contextOptions = context.getOptions(); contextOptions = options; contextOptions.targetColorSpaceOverride = "lin_rec709"; @@ -246,18 +241,19 @@ bool OslShaderRenderTester::runRenderer(const std::string& shaderName, if (shader == nullptr) { log << ">> Failed to generate shader\n"; - return false; + result.success = false; + return result; } CHECK(shader->getSourceCode().length() > 0); std::string oslCmdStr = shader->getSourceCode(); std::string shaderPath; - mx::FilePath outputFilePath = outputPath; + mx::FilePath outputFilePath = item.outputPath; // Note: mkdir will fail if the directory already exists which is ok. { - mx::ScopedTimer ioDir(&profileTimes.languageTimes.ioTime); + mx::ScopedTimer ioDir(&result.languageTimes.ioTime); outputFilePath.createDirectory(true); // Use separate directory for reduced output @@ -273,7 +269,7 @@ bool OslShaderRenderTester::runRenderer(const std::string& shaderName, // Write out osl file if (testOptions.dumpGeneratedCode) { - mx::ScopedTimer ioTimer(&profileTimes.languageTimes.ioTime); + mx::ScopedTimer ioTimer(&result.languageTimes.ioTime); std::ofstream file; file.open(shaderPath + (_useOslCmdStr ? ".oslcmd" : ".osl")); file << shader->getSourceCode(); @@ -293,7 +289,7 @@ bool OslShaderRenderTester::runRenderer(const std::string& shaderName, // Validate compilation if (!_useOslCmdStr) { - mx::ScopedTimer compileTimer(&profileTimes.languageTimes.compileTime); + mx::ScopedTimer compileTimer(&result.languageTimes.compileTime); _renderer->createProgram(shader); } @@ -341,11 +337,11 @@ bool OslShaderRenderTester::runRenderer(const std::string& shaderName, // Validate rendering { - mx::ScopedTimer renderTimer(&profileTimes.languageTimes.renderTime); + mx::ScopedTimer renderTimer(&result.languageTimes.renderTime); _renderer->render(); - if (imageVec) + if (item.imageVec) { - imageVec->push_back(_renderer->captureImage()); + item.imageVec->push_back(_renderer->captureImage()); } } } @@ -376,10 +372,12 @@ bool OslShaderRenderTester::runRenderer(const std::string& shaderName, log << e.what(); } CHECK(validated); + if (!validated) + result.success = false; } } - return true; + return result; } TEST_CASE("Render: OSL TestSuite", "[renderosl]") diff --git a/source/MaterialXTest/MaterialXRenderSlang/RenderSlang.cpp b/source/MaterialXTest/MaterialXRenderSlang/RenderSlang.cpp index 45937248e0..000b7747ac 100644 --- a/source/MaterialXTest/MaterialXRenderSlang/RenderSlang.cpp +++ b/source/MaterialXTest/MaterialXRenderSlang/RenderSlang.cpp @@ -40,16 +40,10 @@ class SlangShaderRenderTester : public RenderUtil::ShaderRenderTester void createRenderer(std::ostream& log) override; - bool runRenderer(const std::string& shaderName, - mx::TypedElementPtr element, - mx::GenContext& context, - mx::DocumentPtr doc, - std::ostream& log, - const GenShaderUtil::TestSuiteOptions& testOptions, - RenderUtil::RenderProfileTimes& profileTimes, - const mx::FileSearchPath& imageSearchPath, - const std::string& outputPath = ".", - mx::ImageVec* imageVec = nullptr) override; + RenderUtil::RenderProfileResult runRenderer( + const RenderUtil::RenderSession& session, + const RenderUtil::RenderItem& item, + mx::GenContext& context) override; bool saveImage(const mx::FilePath& filePath, mx::ConstImagePtr image, bool verticalFlip) const override; @@ -150,20 +144,21 @@ bool SlangShaderRenderTester::saveImage(const mx::FilePath& filePath, mx::ConstI return _renderer->getImageHandler()->saveImage(filePath, image, verticalFlip); } -bool SlangShaderRenderTester::runRenderer(const std::string& shaderName, - mx::TypedElementPtr element, - mx::GenContext& context, - mx::DocumentPtr doc, - std::ostream& log, - const GenShaderUtil::TestSuiteOptions& testOptions, - RenderUtil::RenderProfileTimes& profileTimes, - const mx::FileSearchPath& imageSearchPath, - const std::string& outputPath, - mx::ImageVec* imageVec) +RenderUtil::RenderProfileResult SlangShaderRenderTester::runRenderer( + const RenderUtil::RenderSession& session, + const RenderUtil::RenderItem& item, + mx::GenContext& context) { + RenderUtil::RenderProfileResult result; + const std::string& shaderName = item.shaderName; + mx::DocumentPtr doc = item.doc(); + mx::TypedElementPtr element = item.element; + const GenShaderUtil::TestSuiteOptions& testOptions = session.testOptions; + std::ostream& log = session.log; + std::cout << "Validating Slang rendering for: " << doc->getSourceUri() << std::endl; - mx::ScopedTimer totalGLSLTime(&profileTimes.languageTimes.totalTime); + mx::ScopedTimer totalSlangTime(&result.languageTimes.totalTime); const mx::ShaderGenerator& shadergen = context.getShaderGenerator(); mx::FileSearchPath searchPath = mx::getDefaultDataSearchPath(); @@ -177,13 +172,13 @@ bool SlangShaderRenderTester::runRenderer(const std::string& shaderName, for (auto options : optionsList) { - profileTimes.elementsTested++; + result.elementsTested++; - mx::FilePath outputFilePath = outputPath; + mx::FilePath outputFilePath = item.outputPath; // Note: mkdir will fail if the directory already exists which is ok. { - mx::ScopedTimer ioDir(&profileTimes.languageTimes.ioTime); + mx::ScopedTimer ioDir(&result.languageTimes.ioTime); outputFilePath.createDirectory(true); // Use separate directory for reduced output @@ -198,11 +193,11 @@ bool SlangShaderRenderTester::runRenderer(const std::string& shaderName, mx::ShaderPtr shader; try { - mx::ScopedTimer transpTimer(&profileTimes.languageTimes.transparencyTime); + mx::ScopedTimer transpTimer(&result.languageTimes.transparencyTime); options.hwTransparency = mx::isTransparentSurface(element, shadergen.getTarget()); transpTimer.endTimer(); - mx::ScopedTimer generationTimer(&profileTimes.languageTimes.generationTime); + mx::ScopedTimer generationTimer(&result.languageTimes.generationTime); mx::GenOptions& contextOptions = context.getOptions(); contextOptions = options; contextOptions.targetColorSpaceOverride = "lin_rec709"; @@ -219,7 +214,8 @@ bool SlangShaderRenderTester::runRenderer(const std::string& shaderName, if (shader == nullptr) { log << ">> Failed to generate shader\n"; - return false; + result.success = false; + return result; } const std::string& vertexSourceCode = shader->getSourceCode(mx::Stage::VERTEX); const std::string& pixelSourceCode = shader->getSourceCode(mx::Stage::PIXEL); @@ -228,7 +224,7 @@ bool SlangShaderRenderTester::runRenderer(const std::string& shaderName, if (testOptions.dumpGeneratedCode) { - mx::ScopedTimer dumpTimer(&profileTimes.languageTimes.ioTime); + mx::ScopedTimer dumpTimer(&result.languageTimes.ioTime); std::ofstream file; file.open(shaderPath + "_vs.slang"); file << vertexSourceCode; @@ -279,14 +275,14 @@ bool SlangShaderRenderTester::runRenderer(const std::string& shaderName, _renderer->setLightHandler(isShader ? _lightHandler : nullptr); { - mx::ScopedTimer compileTimer(&profileTimes.languageTimes.compileTime); + mx::ScopedTimer compileTimer(&result.languageTimes.compileTime); _renderer->createProgram(shader); _renderer->validateInputs(); } if (testOptions.dumpUniformsAndAttributes) { - mx::ScopedTimer printTimer(&profileTimes.languageTimes.ioTime); + mx::ScopedTimer printTimer(&result.languageTimes.ioTime); log << "* Uniform:" << std::endl; program->printUniforms(log); log << "* Attributes:" << std::endl; @@ -345,8 +341,8 @@ bool SlangShaderRenderTester::runRenderer(const std::string& shaderName, int supersampleFactor = testOptions.enableReferenceQuality ? 8 : 1; { - mx::ScopedTimer renderTimer(&profileTimes.languageTimes.renderTime); - _renderer->getImageHandler()->setSearchPath(imageSearchPath); + mx::ScopedTimer renderTimer(&result.languageTimes.renderTime); + _renderer->getImageHandler()->setSearchPath(item.imageSearchPath); unsigned int width = (unsigned int) testOptions.renderSize[0] * supersampleFactor; unsigned int height = (unsigned int) testOptions.renderSize[1] * supersampleFactor; _renderer->setSize(width, height); @@ -354,7 +350,7 @@ bool SlangShaderRenderTester::runRenderer(const std::string& shaderName, } { - mx::ScopedTimer ioTimer(&profileTimes.languageTimes.imageSaveTime); + mx::ScopedTimer ioTimer(&result.languageTimes.imageSaveTime); std::string fileName = shaderPath + "_slang.png"; mx::ImagePtr image = _renderer->captureImage(); if (image) @@ -364,9 +360,9 @@ bool SlangShaderRenderTester::runRenderer(const std::string& shaderName, image = image->applyBoxDownsample(supersampleFactor); } _renderer->getImageHandler()->saveImage(fileName, image); - if (imageVec) + if (item.imageVec) { - imageVec->push_back(image); + item.imageVec->push_back(image); } } } @@ -397,9 +393,11 @@ bool SlangShaderRenderTester::runRenderer(const std::string& shaderName, WARN(std::string(e.what()) + " in " + shaderPath); } CHECK(validated); + if (!validated) + result.success = false; } } - return true; + return result; } TEST_CASE("Render: Slang TestSuite", "[renderslang]")