From c3d869482f031a792c1c54c0f78d4cda47c6da09 Mon Sep 17 00:00:00 2001 From: Chainsawkitten Date: Fri, 23 Jun 2023 15:15:05 +0200 Subject: [PATCH] Make unsupported tests clearer Some unit tests rely on optional device features (eg. conservative rasterization). Add a way for such tests to report that they are unsupported (rather than either a success or a failure) when run on a device without the required feature. --- tests/Framework/TestSuite.cpp | 22 +++++++++--- tests/Framework/TestSuite.hpp | 30 ++++++++++++---- tests/Video/ComputeTests.cpp | 16 ++++----- tests/Video/DrawTests.cpp | 63 +++++++++++++++++----------------- tests/Video/SwapChainTests.cpp | 4 +-- tests/Video/VideoSuite.hpp | 38 ++++++++++---------- tests/main.cpp | 5 +-- 7 files changed, 105 insertions(+), 73 deletions(-) diff --git a/tests/Framework/TestSuite.cpp b/tests/Framework/TestSuite.cpp index c0e2e45a..94a2b1fe 100644 --- a/tests/Framework/TestSuite.cpp +++ b/tests/Framework/TestSuite.cpp @@ -4,6 +4,10 @@ using namespace std; +Result ToResult(bool success) { + return success ? Result::SUCCESS : Result::FAILURE; +} + TestSuite::TestSuite(const string& name) { this->name = name; } @@ -18,19 +22,27 @@ void TestSuite::Run(const string& suiteName, const string& testName, Results& re if (testName.empty() || test.name == testName) { cout << test.name << ": "; - if (test.function(test.data)) { + Result result = test.function(test.data); + switch (result) { + case Result::SUCCESS: cout << "Success\n"; results.success++; - } else { + break; + case Result::FAILURE: cout << "Fail\n"; results.failure++; + break; + case Result::UNSUPPORTED: + cout << "Unsupported\n"; + results.unsupported++; + break; } } else { - results.notRun++; + results.skipped++; } } } else { - results.notRun += tests.size(); + results.skipped += tests.size(); } } @@ -44,7 +56,7 @@ void TestSuite::Print() const { } } -void TestSuite::AddTest(const string& name, bool (*function)(void*), void* data) { +void TestSuite::AddTest(const string& name, Result (*function)(void*), void* data) { Test test; test.name = name; test.function = function; diff --git a/tests/Framework/TestSuite.hpp b/tests/Framework/TestSuite.hpp index 16de9160..53c78712 100644 --- a/tests/Framework/TestSuite.hpp +++ b/tests/Framework/TestSuite.hpp @@ -3,6 +3,21 @@ #include #include +/// The result of a test. +enum class Result { + SUCCESS, ///< Test executed successfully. + FAILURE, ///< Test failed. + UNSUPPORTED ///< Test requires functionality not supported on this device. +}; + +/// Converts a bool to a Result enum. +/** + * @param success Whether the test was a success. + * + * @return Either SUCCESS or FAILURE. + */ +Result ToResult(bool success); + /// A suite of tests, testing one module. /** * @todo Linked list of suites. @@ -11,14 +26,17 @@ class TestSuite { public: /// The results of a run. struct Results { + /// The number of successful tests. + unsigned int success = 0; + /// The number of failed tests. unsigned int failure = 0; - /// The number of successful tests. - unsigned int success = 0; + /// The number of unsupported tests. + unsigned int unsupported = 0; - /// The number of tests that weren't run. - unsigned int notRun = 0; + /// The number of tests that were skipped. + unsigned int skipped = 0; }; /// Create a new test suite. @@ -48,14 +66,14 @@ class TestSuite { * @param function The function containing the test code. * @param data Data for the function to process. */ - void AddTest(const std::string& name, bool (*function)(void*), void* data = nullptr); + void AddTest(const std::string& name, Result (*function)(void*), void* data = nullptr); private: std::string name; struct Test { std::string name; - bool (*function)(void*); + Result (*function)(void*); void* data; }; std::vector tests; diff --git a/tests/Video/ComputeTests.cpp b/tests/Video/ComputeTests.cpp index 4ee72066..2e63a93d 100644 --- a/tests/Video/ComputeTests.cpp +++ b/tests/Video/ComputeTests.cpp @@ -36,7 +36,7 @@ using namespace Video; static const unsigned int imageSize = 64; -bool ComputeSetBuffer(void* data) { +Result ComputeSetBuffer(void* data) { assert(data != nullptr); LowLevelRenderer* lowLevelRenderer = *static_cast(data); @@ -119,10 +119,10 @@ bool ComputeSetBuffer(void* data) { delete computeShaderProgram; delete computeShader; - return result; + return ToResult(result); } -bool ComputeVertexBuffer(void* data) { +Result ComputeVertexBuffer(void* data) { assert(data != nullptr); LowLevelRenderer* lowLevelRenderer = *static_cast(data); @@ -218,10 +218,10 @@ bool ComputeVertexBuffer(void* data) { delete computeShaderProgram; delete computeShader; - return result; + return ToResult(result); } -bool ComputeMultipleBuffers(void* data) { +Result ComputeMultipleBuffers(void* data) { assert(data != nullptr); LowLevelRenderer* lowLevelRenderer = *static_cast(data); @@ -301,10 +301,10 @@ bool ComputeMultipleBuffers(void* data) { delete computeShaderProgram; delete computeShader; - return result; + return ToResult(result); } -bool ComputeClearBuffer(void* data) { +Result ComputeClearBuffer(void* data) { assert(data != nullptr); LowLevelRenderer* lowLevelRenderer = *static_cast(data); @@ -389,5 +389,5 @@ bool ComputeClearBuffer(void* data) { delete computeShaderProgram; delete computeShader; - return result; + return ToResult(result); } diff --git a/tests/Video/DrawTests.cpp b/tests/Video/DrawTests.cpp index 9bbe78ab..55524432 100644 --- a/tests/Video/DrawTests.cpp +++ b/tests/Video/DrawTests.cpp @@ -61,7 +61,7 @@ using namespace Video; static const unsigned int imageSize = 64; -bool DrawTriangle(void* data) { +Result DrawTriangle(void* data) { assert(data != nullptr); LowLevelRenderer* lowLevelRenderer = *static_cast(data); @@ -115,10 +115,10 @@ bool DrawTriangle(void* data) { delete vertexShader; delete fragmentShader; - return result; + return ToResult(result); } -bool DrawVertexTriangle(void* data) { +Result DrawVertexTriangle(void* data) { assert(data != nullptr); LowLevelRenderer* lowLevelRenderer = *static_cast(data); @@ -209,10 +209,10 @@ bool DrawVertexTriangle(void* data) { delete vertexShader; delete fragmentShader; - return result; + return ToResult(result); } -bool DrawTexturedTriangle(void* data) { +Result DrawTexturedTriangle(void* data) { assert(data != nullptr); LowLevelRenderer* lowLevelRenderer = *static_cast(data); @@ -312,10 +312,10 @@ bool DrawTexturedTriangle(void* data) { delete vertexShader; delete fragmentShader; - return result; + return ToResult(result); } -bool DrawQuad(void* data) { +Result DrawQuad(void* data) { assert(data != nullptr); LowLevelRenderer* lowLevelRenderer = *static_cast(data); @@ -419,10 +419,10 @@ bool DrawQuad(void* data) { delete vertexShader; delete fragmentShader; - return result; + return ToResult(result); } -bool DrawTriangles(void* data) { +Result DrawTriangles(void* data) { assert(data != nullptr); LowLevelRenderer* lowLevelRenderer = *static_cast(data); @@ -527,10 +527,10 @@ bool DrawTriangles(void* data) { delete vertexShader; delete fragmentShader; - return result; + return ToResult(result); } -bool DrawPushTriangles(void* data) { +Result DrawPushTriangles(void* data) { assert(data != nullptr); LowLevelRenderer* lowLevelRenderer = *static_cast(data); @@ -632,10 +632,10 @@ bool DrawPushTriangles(void* data) { delete vertexShader; delete fragmentShader; - return result; + return ToResult(result); } -bool DrawStorageTriangle(void* data) { +Result DrawStorageTriangle(void* data) { assert(data != nullptr); LowLevelRenderer* lowLevelRenderer = *static_cast(data); @@ -705,10 +705,10 @@ bool DrawStorageTriangle(void* data) { delete vertexShader; delete fragmentShader; - return result; + return ToResult(result); } -bool InvertColors(void* data) { +Result InvertColors(void* data) { assert(data != nullptr); LowLevelRenderer* lowLevelRenderer = *static_cast(data); @@ -840,10 +840,10 @@ bool InvertColors(void* data) { delete invertVertexShader; delete invertFragmentShader; - return result; + return ToResult(result); } -bool DrawMipmappedTriangle(void* data) { +Result DrawMipmappedTriangle(void* data) { assert(data != nullptr); LowLevelRenderer* lowLevelRenderer = *static_cast(data); @@ -940,10 +940,10 @@ bool DrawMipmappedTriangle(void* data) { delete vertexShader; delete fragmentShader; - return result; + return ToResult(result); } -bool DepthPrePass(void* data) { +Result DepthPrePass(void* data) { assert(data != nullptr); LowLevelRenderer* lowLevelRenderer = *static_cast(data); @@ -1057,17 +1057,17 @@ bool DepthPrePass(void* data) { delete vertexShader; delete fragmentShader; - return result; + return ToResult(result); } -bool DrawLines(void* data) { +Result DrawLines(void* data) { assert(data != nullptr); LowLevelRenderer* lowLevelRenderer = *static_cast(data); // This test requires fillModeNonSolid if (!lowLevelRenderer->GetOptionalFeatures().fillModeNonSolid) { - return false; + return Result::UNSUPPORTED; } // Create render texture. @@ -1157,10 +1157,10 @@ bool DrawLines(void* data) { delete vertexShader; delete fragmentShader; - return result; + return ToResult(result); } -bool Attachmentless(void* data) { +Result Attachmentless(void* data) { assert(data != nullptr); LowLevelRenderer* lowLevelRenderer = *static_cast(data); @@ -1236,17 +1236,18 @@ bool Attachmentless(void* data) { delete readFragmentShader; delete storageBuffer; - return result; + return ToResult(result); } -bool ConservativeRasterization(void* data) { +Result ConservativeRasterization(void* data) { assert(data != nullptr); LowLevelRenderer* lowLevelRenderer = *static_cast(data); // This is an optional feature. - if (!lowLevelRenderer->GetOptionalFeatures().conservativeRasterization) - return true; + if (!lowLevelRenderer->GetOptionalFeatures().conservativeRasterization) { + return Result::UNSUPPORTED; + } // Create render texture. Texture* renderTexture = lowLevelRenderer->CreateRenderTarget(glm::uvec2(imageSize, imageSize), Texture::Format::R8G8B8A8); @@ -1298,10 +1299,10 @@ bool ConservativeRasterization(void* data) { delete vertexShader; delete fragmentShader; - return result; + return ToResult(result); } -bool DepthClamping(void* data) { +Result DepthClamping(void* data) { assert(data != nullptr); LowLevelRenderer* lowLevelRenderer = *static_cast(data); @@ -1417,5 +1418,5 @@ bool DepthClamping(void* data) { delete vertexShader; delete fragmentShader; - return result; + return ToResult(result); } diff --git a/tests/Video/SwapChainTests.cpp b/tests/Video/SwapChainTests.cpp index bd94172b..58061b76 100644 --- a/tests/Video/SwapChainTests.cpp +++ b/tests/Video/SwapChainTests.cpp @@ -21,7 +21,7 @@ using namespace Video; -bool MultipleFrames(void* data) { +Result MultipleFrames(void* data) { assert(data != nullptr); const uint32_t frameCount = 120; @@ -133,5 +133,5 @@ bool MultipleFrames(void* data) { delete vertexShader; delete fragmentShader; - return true; + return Result::SUCCESS; } diff --git a/tests/Video/VideoSuite.hpp b/tests/Video/VideoSuite.hpp index fc970040..c8545407 100644 --- a/tests/Video/VideoSuite.hpp +++ b/tests/Video/VideoSuite.hpp @@ -44,7 +44,7 @@ class VideoSuite : public TestSuite { * * @return Whether the test was successful. */ -bool DrawTriangle(void* data); +Result DrawTriangle(void* data); /// Draw a colored triangle from a vertex buffer. /** @@ -52,7 +52,7 @@ bool DrawTriangle(void* data); * * @return Whether the test was successful. */ -bool DrawVertexTriangle(void* data); +Result DrawVertexTriangle(void* data); /// Draw a textured triangle. /** @@ -60,7 +60,7 @@ bool DrawVertexTriangle(void* data); * * @return Whether the test was successful. */ -bool DrawTexturedTriangle(void* data); +Result DrawTexturedTriangle(void* data); /// Draw a quad using a vertex and index buffer. /** @@ -68,7 +68,7 @@ bool DrawTexturedTriangle(void* data); * * @return Whether the test was successful. */ -bool DrawQuad(void* data); +Result DrawQuad(void* data); /// Draw two colored triangles, offset by a model matrix in uniform buffer and using a depth buffer. /** @@ -76,7 +76,7 @@ bool DrawQuad(void* data); * * @return Whether the test was successful. */ -bool DrawTriangles(void* data); +Result DrawTriangles(void* data); /// Draw two colored triangles, offset by a model matrix in push constants. /** @@ -84,7 +84,7 @@ bool DrawTriangles(void* data); * * @return Whether the test was successful. */ -bool DrawPushTriangles(void* data); +Result DrawPushTriangles(void* data); /// Draw a triangle using vertex pulling from a storage buffer. /** @@ -92,7 +92,7 @@ bool DrawPushTriangles(void* data); * * @return Whether the test was successful. */ -bool DrawStorageTriangle(void* data); +Result DrawStorageTriangle(void* data); /// Draw two triangles, then invert the colors. /** @@ -100,7 +100,7 @@ bool DrawStorageTriangle(void* data); * * @return Whether the test was successful. */ -bool InvertColors(void* data); +Result InvertColors(void* data); /// Draw a mipmapped triangle. /** @@ -108,7 +108,7 @@ bool InvertColors(void* data); * * @return Whether the test was successful. */ -bool DrawMipmappedTriangle(void* data); +Result DrawMipmappedTriangle(void* data); /// Use a depth pre-pass to occlude some fragments. /** @@ -116,7 +116,7 @@ bool DrawMipmappedTriangle(void* data); * * @return Whether the test was successful. */ -bool DepthPrePass(void* data); +Result DepthPrePass(void* data); /// Draw lines with width. /** @@ -124,7 +124,7 @@ bool DepthPrePass(void* data); * * @return Whether the test was successful. */ -bool DrawLines(void* data); +Result DrawLines(void* data); /// Test attachmentless rendering. /** @@ -132,7 +132,7 @@ bool DrawLines(void* data); * * @return Whether the test was successful. */ -bool Attachmentless(void* data); +Result Attachmentless(void* data); /// Test conservative rasterization. /** @@ -140,7 +140,7 @@ bool Attachmentless(void* data); * * @return Whether the test was successful. */ -bool ConservativeRasterization(void* data); +Result ConservativeRasterization(void* data); /// Test depth clamping. /** @@ -148,7 +148,7 @@ bool ConservativeRasterization(void* data); * * @return Whether the test was successful. */ -bool DepthClamping(void* data); +Result DepthClamping(void* data); /// Test rendering multiple frames. /** @@ -156,7 +156,7 @@ bool DepthClamping(void* data); * * @return Whether the test was successful. */ -bool MultipleFrames(void* data); +Result MultipleFrames(void* data); /// Test setting buffer contents in a compute shader. /** @@ -164,7 +164,7 @@ bool MultipleFrames(void* data); * * @return Whether the test was successful. */ -bool ComputeSetBuffer(void* data); +Result ComputeSetBuffer(void* data); /// Test writing vertex buffer contents in a compute shader. /** @@ -172,7 +172,7 @@ bool ComputeSetBuffer(void* data); * * @return Whether the test was successful. */ -bool ComputeVertexBuffer(void* data); +Result ComputeVertexBuffer(void* data); /// Test using multiple storage buffers in a compute shaders. /** @@ -180,7 +180,7 @@ bool ComputeVertexBuffer(void* data); * * @return Whether the test was successful. */ -bool ComputeMultipleBuffers(void* data); +Result ComputeMultipleBuffers(void* data); /// Test using ClearBuffer. /** @@ -188,4 +188,4 @@ bool ComputeMultipleBuffers(void* data); * * @return Whether the test was successful. */ -bool ComputeClearBuffer(void* data); +Result ComputeClearBuffer(void* data); diff --git a/tests/main.cpp b/tests/main.cpp index cf50113d..790554fa 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -61,12 +61,13 @@ int main(int argc, char* argv[]) { videoSuite.Shutdown(); // Print summary of results. - const unsigned int total = results.success + results.failure + results.notRun; + const unsigned int total = results.success + results.unsupported + results.failure + results.skipped; cout << "\n\n" << "Results:\n" << " Success: " << results.success << " / " << total << "\n" + << " Unsupported: " << results.unsupported << " / " << total << "\n" << " Failure: " << results.failure << " / " << total << "\n" - << " Not run: " << results.notRun << " / " << total << "\n"; + << " Skipped: " << results.skipped << " / " << total << "\n"; if (results.failure > 0) { return 1;