Make unsupported tests clearer

Some unit tests rely on optional device features (eg. conservative rasterization). Add a way for such tests to report that they are unsupported (rather than either a success or a failure) when run on a device without the required feature.
Chainsawkitten · Jun 23, 2023 · 030e69f · 030e69f
1 parent dc2261d
commit 030e69f
Show file tree

Hide file tree

Showing 7 changed files with 105 additions and 73 deletions.
diff --git a/tests/Framework/TestSuite.cpp b/tests/Framework/TestSuite.cpp
@@ -4,6 +4,10 @@
 
 using namespace std;
 
+Result ToResult(bool success) {
+    return success ? Result::SUCCESS : Result::FAILURE;
+}
+
 TestSuite::TestSuite(const string& name) {
     this->name = name;
 }
@@ -18,19 +22,27 @@ void TestSuite::Run(const string& suiteName, const string& testName, Results& re
             if (testName.empty() || test.name == testName) {
                 cout << test.name << ": ";
 
-                if (test.function(test.data)) {
+                Result result = test.function(test.data);
+                switch (result) {
+                case Result::SUCCESS:
                     cout << "Success\n";
                     results.success++;
-                } else {
+                    break;
+                case Result::FAILURE:
                     cout << "Fail\n";
                     results.failure++;
+                    break;
+                case Result::UNSUPPORTED:
+                    cout << "Unsupported\n";
+                    results.unsupported++;
+                    break;
                 }
             } else {
-                results.notRun++;
+                results.skipped++;
             }
         }
     } else {
-        results.notRun += tests.size();
+        results.skipped += tests.size();
     }
 }
 
@@ -44,7 +56,7 @@ void TestSuite::Print() const {
     }
 }
 
-void TestSuite::AddTest(const string& name, bool (*function)(void*), void* data) {
+void TestSuite::AddTest(const string& name, Result (*function)(void*), void* data) {
     Test test;
     test.name = name;
     test.function = function;

diff --git a/tests/Framework/TestSuite.hpp b/tests/Framework/TestSuite.hpp
@@ -3,6 +3,21 @@
 #include <string>
 #include <vector>
 
+/// The result of a test.
+enum class Result {
+    SUCCESS, ///< Test executed successfully.
+    FAILURE, ///< Test failed.
+    UNSUPPORTED ///< Test requires functionality not supported on this device.
+};
+
+/// Converts a bool to a Result enum.
+/**
+ * @param success Whether the test was a success.
+ *
+ * @return Either SUCCESS or FAILURE.
+ */
+Result ToResult(bool success);
+
 /// A suite of tests, testing one module.
 /**
  * @todo Linked list of suites.
@@ -11,14 +26,17 @@ class TestSuite {
   public:
     /// The results of a run.
     struct Results {
+        /// The number of successful tests.
+        unsigned int success = 0;
+
         /// The number of failed tests.
         unsigned int failure = 0;
 
-        /// The number of successful tests.
-        unsigned int success = 0;
+        /// The number of unsupported tests.
+        unsigned int unsupported = 0;
 
-        /// The number of tests that weren't run.
-        unsigned int notRun = 0;
+        /// The number of tests that were skipped.
+        unsigned int skipped = 0;
     };
 
     /// Create a new test suite.
@@ -48,14 +66,14 @@ class TestSuite {
      * @param function The function containing the test code.
      * @param data Data for the function to process.
      */
-    void AddTest(const std::string& name, bool (*function)(void*), void* data = nullptr);
+    void AddTest(const std::string& name, Result (*function)(void*), void* data = nullptr);
 
   private:
     std::string name;
 
     struct Test {
         std::string name;
-        bool (*function)(void*);
+        Result (*function)(void*);
         void* data;
     };
     std::vector<Test> tests;

diff --git a/tests/Video/ComputeTests.cpp b/tests/Video/ComputeTests.cpp
@@ -36,7 +36,7 @@ using namespace Video;
 
 static const unsigned int imageSize = 64;
 
-bool ComputeSetBuffer(void* data) {
+Result ComputeSetBuffer(void* data) {
     assert(data != nullptr);
 
     LowLevelRenderer* lowLevelRenderer = *static_cast<LowLevelRenderer**>(data);
@@ -119,10 +119,10 @@ bool ComputeSetBuffer(void* data) {
     delete computeShaderProgram;
     delete computeShader;
 
-    return result;
+    return ToResult(result);
 }
 
-bool ComputeVertexBuffer(void* data) {
+Result ComputeVertexBuffer(void* data) {
     assert(data != nullptr);
 
     LowLevelRenderer* lowLevelRenderer = *static_cast<LowLevelRenderer**>(data);
@@ -218,10 +218,10 @@ bool ComputeVertexBuffer(void* data) {
     delete computeShaderProgram;
     delete computeShader;
 
-    return result;
+    return ToResult(result);
 }
 
-bool ComputeMultipleBuffers(void* data) {
+Result ComputeMultipleBuffers(void* data) {
     assert(data != nullptr);
 
     LowLevelRenderer* lowLevelRenderer = *static_cast<LowLevelRenderer**>(data);
@@ -301,10 +301,10 @@ bool ComputeMultipleBuffers(void* data) {
     delete computeShaderProgram;
     delete computeShader;
 
-    return result;
+    return ToResult(result);
 }
 
-bool ComputeClearBuffer(void* data) {
+Result ComputeClearBuffer(void* data) {
     assert(data != nullptr);
 
     LowLevelRenderer* lowLevelRenderer = *static_cast<LowLevelRenderer**>(data);
@@ -389,5 +389,5 @@ bool ComputeClearBuffer(void* data) {
     delete computeShaderProgram;
     delete computeShader;
 
-    return result;
+    return ToResult(result);
 }
diff --git a/tests/Video/DrawTests.cpp b/tests/Video/DrawTests.cpp
@@ -61,7 +61,7 @@ using namespace Video;
 
 static const unsigned int imageSize = 64;
 
-bool DrawTriangle(void* data) {
+Result DrawTriangle(void* data) {
     assert(data != nullptr);
 
     LowLevelRenderer* lowLevelRenderer = *static_cast<LowLevelRenderer**>(data);
@@ -115,10 +115,10 @@ bool DrawTriangle(void* data) {
     delete vertexShader;
     delete fragmentShader;
 
-    return result;
+    return ToResult(result);
 }
 
-bool DrawVertexTriangle(void* data) {
+Result DrawVertexTriangle(void* data) {
     assert(data != nullptr);
 
     LowLevelRenderer* lowLevelRenderer = *static_cast<LowLevelRenderer**>(data);
@@ -209,10 +209,10 @@ bool DrawVertexTriangle(void* data) {
     delete vertexShader;
     delete fragmentShader;
 
-    return result;
+    return ToResult(result);
 }
 
-bool DrawTexturedTriangle(void* data) {
+Result DrawTexturedTriangle(void* data) {
     assert(data != nullptr);
 
     LowLevelRenderer* lowLevelRenderer = *static_cast<LowLevelRenderer**>(data);
@@ -312,10 +312,10 @@ bool DrawTexturedTriangle(void* data) {
     delete vertexShader;
     delete fragmentShader;
 
-    return result;
+    return ToResult(result);
 }
 
-bool DrawQuad(void* data) {
+Result DrawQuad(void* data) {
     assert(data != nullptr);
 
     LowLevelRenderer* lowLevelRenderer = *static_cast<LowLevelRenderer**>(data);
@@ -419,10 +419,10 @@ bool DrawQuad(void* data) {
     delete vertexShader;
     delete fragmentShader;
 
-    return result;
+    return ToResult(result);
 }
 
-bool DrawTriangles(void* data) {
+Result DrawTriangles(void* data) {
     assert(data != nullptr);
 
     LowLevelRenderer* lowLevelRenderer = *static_cast<LowLevelRenderer**>(data);
@@ -527,10 +527,10 @@ bool DrawTriangles(void* data) {
     delete vertexShader;
     delete fragmentShader;
 
-    return result;
+    return ToResult(result);
 }
 
-bool DrawPushTriangles(void* data) {
+Result DrawPushTriangles(void* data) {
     assert(data != nullptr);
 
     LowLevelRenderer* lowLevelRenderer = *static_cast<LowLevelRenderer**>(data);
@@ -632,10 +632,10 @@ bool DrawPushTriangles(void* data) {
     delete vertexShader;
     delete fragmentShader;
 
-    return result;
+    return ToResult(result);
 }
 
-bool DrawStorageTriangle(void* data) {
+Result DrawStorageTriangle(void* data) {
     assert(data != nullptr);
 
     LowLevelRenderer* lowLevelRenderer = *static_cast<LowLevelRenderer**>(data);
@@ -705,10 +705,10 @@ bool DrawStorageTriangle(void* data) {
     delete vertexShader;
     delete fragmentShader;
 
-    return result;
+    return ToResult(result);
 }
 
-bool InvertColors(void* data) {
+Result InvertColors(void* data) {
     assert(data != nullptr);
 
     LowLevelRenderer* lowLevelRenderer = *static_cast<LowLevelRenderer**>(data);
@@ -840,10 +840,10 @@ bool InvertColors(void* data) {
     delete invertVertexShader;
     delete invertFragmentShader;
 
-    return result;
+    return ToResult(result);
 }
 
-bool DrawMipmappedTriangle(void* data) {
+Result DrawMipmappedTriangle(void* data) {
     assert(data != nullptr);
 
     LowLevelRenderer* lowLevelRenderer = *static_cast<LowLevelRenderer**>(data);
@@ -940,10 +940,10 @@ bool DrawMipmappedTriangle(void* data) {
     delete vertexShader;
     delete fragmentShader;
 
-    return result;
+    return ToResult(result);
 }
 
-bool DepthPrePass(void* data) {
+Result DepthPrePass(void* data) {
     assert(data != nullptr);
 
     LowLevelRenderer* lowLevelRenderer = *static_cast<LowLevelRenderer**>(data);
@@ -1057,17 +1057,17 @@ bool DepthPrePass(void* data) {
     delete vertexShader;
     delete fragmentShader;
 
-    return result;
+    return ToResult(result);
 }
 
-bool DrawLines(void* data) {
+Result DrawLines(void* data) {
     assert(data != nullptr);
 
     LowLevelRenderer* lowLevelRenderer = *static_cast<LowLevelRenderer**>(data);
 
     // This test requires fillModeNonSolid
     if (!lowLevelRenderer->GetOptionalFeatures().fillModeNonSolid) {
-        return false;
+        return Result::UNSUPPORTED;
     }
 
     // Create render texture.
@@ -1157,10 +1157,10 @@ bool DrawLines(void* data) {
     delete vertexShader;
     delete fragmentShader;
 
-    return result;
+    return ToResult(result);
 }
 
-bool Attachmentless(void* data) {
+Result Attachmentless(void* data) {
     assert(data != nullptr);
 
     LowLevelRenderer* lowLevelRenderer = *static_cast<LowLevelRenderer**>(data);
@@ -1236,17 +1236,18 @@ bool Attachmentless(void* data) {
     delete readFragmentShader;
     delete storageBuffer;
 
-    return result;
+    return ToResult(result);
 }
 
-bool ConservativeRasterization(void* data) {
+Result ConservativeRasterization(void* data) {
     assert(data != nullptr);
 
     LowLevelRenderer* lowLevelRenderer = *static_cast<LowLevelRenderer**>(data);
 
     // This is an optional feature.
-    if (!lowLevelRenderer->GetOptionalFeatures().conservativeRasterization)
-        return true;
+    if (!lowLevelRenderer->GetOptionalFeatures().conservativeRasterization) {
+        return Result::UNSUPPORTED;
+    }
 
     // Create render texture.
     Texture* renderTexture = lowLevelRenderer->CreateRenderTarget(glm::uvec2(imageSize, imageSize), Texture::Format::R8G8B8A8);
@@ -1298,10 +1299,10 @@ bool ConservativeRasterization(void* data) {
     delete vertexShader;
     delete fragmentShader;
 
-    return result;
+    return ToResult(result);
 }
 
-bool DepthClamping(void* data) {
+Result DepthClamping(void* data) {
     assert(data != nullptr);
 
     LowLevelRenderer* lowLevelRenderer = *static_cast<LowLevelRenderer**>(data);
@@ -1417,5 +1418,5 @@ bool DepthClamping(void* data) {
     delete vertexShader;
     delete fragmentShader;
 
-    return result;
+    return ToResult(result);
 }
diff --git a/tests/Video/SwapChainTests.cpp b/tests/Video/SwapChainTests.cpp
@@ -21,7 +21,7 @@
 
 using namespace Video;
 
-bool MultipleFrames(void* data) {
+Result MultipleFrames(void* data) {
     assert(data != nullptr);
 
     const uint32_t frameCount = 120;
@@ -133,5 +133,5 @@ bool MultipleFrames(void* data) {
     delete vertexShader;
     delete fragmentShader;
 
-    return true;
+    return Result::SUCCESS;
 }