Merge pull request #13 from Themaister/windows-seh

Add SEH exception handler for Windows.

README.md

@@ -217,6 +217,12 @@ This can be quite slow for application with lots of pipelines, so only use it if
 This only works on Linux. A SIGSEGV handler is registered, and the state is serialized to disk in the segfault handler.
 This is a bit sketchy, but should work well if drivers are crashing on pipeline creation (or just crashing in general).
 
+On Windows, all `vkCreate*Pipelines` calls are always wrapped in SEH-style __try/__except blocks, which will catch access violations
+specifically inside those calls. If an access violation is triggered, a safety serialization is performed,
+a message box will appear, notifying user about this,
+and immediately terminate the process after. This functionality however, is only enabled when building with MSVC,
+as MinGW does not readily support the __try/__except extension. Patches welcome!
+
 #### `export FOSSILIZE_DUMP_PATH=/my/custom/path`
 
 Custom file path for capturing state.

layer/device.cpp

@@ -136,7 +136,7 @@ void Device::installSegfaultHandler()
 }
 #endif
 
-void Device::serializeToPath(const std::string &path)
+bool Device::serializeToPath(const std::string &path)
 {
 	try
 	{
@@ -148,15 +148,18 @@ void Device::serializeToPath(const std::string &path)
 				LOGE("Failed to write serialized state to disk.\n");
 			fclose(file);
 			LOGI("Serialized to \"%s\".\n", path.c_str());
+			return true;
 		}
 		else
 		{
 			LOGE("Failed to open file for writing: \"%s\".\n", path.c_str());
+			return false;
 		}
 	}
 	catch (const std::exception &e)
 	{
 		LOGE("Failed to serialize: \"%s\".\n", e.what());
+		return false;
 	}
 }
 

layer/device.hpp

@@ -44,7 +44,7 @@ class Device
 		return recorder;
 	}
 
-	void serializeToPath(const std::string &path);
+	bool serializeToPath(const std::string &path);
 	const std::string &getSerializationPath() const
 	{
 		return serializationPath;
@@ -55,6 +55,11 @@ class Device
 		return paranoidMode;
 	}
 
+	VkDevice getDevice() const
+	{
+		return device;
+	}
+
 private:
 	VkPhysicalDevice gpu = VK_NULL_HANDLE;
 	VkDevice device = VK_NULL_HANDLE;

layer/dispatch.cpp

@@ -26,6 +26,12 @@
 #include "instance.hpp"
 #include <mutex>
 
+#ifdef _MSC_VER // For SEH access violation handling.
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <excpt.h>
+#endif
+
 using namespace std;
 
 namespace Fossilize
@@ -96,6 +102,65 @@ static VKAPI_ATTR void VKAPI_CALL DestroyInstance(VkInstance instance, const VkA
 	destroyLayerData(key, instanceData);
 }
 
+#ifdef _MSC_VER
+static int filterSEHException(int code)
+{
+	return code == EXCEPTION_ACCESS_VIOLATION ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH;
+}
+#endif
+
+static VkResult createGraphicsPipeline(Device *device, VkPipelineCache pipelineCache,
+                                       const VkGraphicsPipelineCreateInfo *pCreateInfo,
+                                       const VkAllocationCallbacks *pCallbacks,
+                                       VkPipeline *pipeline)
+{
+#ifdef _MSC_VER // Dunno how to do SIGSEGV handling on Windows, so ad-hoc SEH it is. This isn't supported on MinGW, so just MSVC for now.
+	__try
+#endif
+	{
+		return device->getTable()->CreateGraphicsPipelines(device->getDevice(), pipelineCache, 1, pCreateInfo, pCallbacks, pipeline);
+	}
+#ifdef _MSC_VER
+	__except (filterSEHException(GetExceptionCode()))
+	{
+		LOGE("Caught access violation in vkCreateGraphicsPipelines(), safety serialization before terminating ...\n");
+		bool success = device->serializeToPath(device->getSerializationPath());
+
+		if (success)
+			MessageBoxA(nullptr, "vkCreateGraphicsPipelines() triggered an access violation, the offending state has been serialized.", "CreateGraphicsPipeline Access Violation", 0);
+		else
+			MessageBoxA(nullptr, "vkCreateGraphicsPipelines() triggered an access violation, but the offending state failed to be serialized.", "CreateGraphicsPipeline Access Violation", 0);
+		std::terminate();
+	}
+#endif
+}
+
+static VkResult createComputePipeline(Device *device, VkPipelineCache pipelineCache,
+                                      const VkComputePipelineCreateInfo *pCreateInfo,
+                                      const VkAllocationCallbacks *pCallbacks,
+                                      VkPipeline *pipeline)
+{
+#ifdef _MSC_VER // Dunno how to do SIGSEGV handling on Windows, so ad-hoc SEH it is. This isn't supported on MinGW, so just MSVC for now.
+	__try
+#endif
+	{
+		return device->getTable()->CreateComputePipelines(device->getDevice(), pipelineCache, 1, pCreateInfo, pCallbacks, pipeline);
+	}
+#ifdef _MSC_VER
+	__except (filterSEHException(GetExceptionCode()))
+	{
+		LOGE("Caught access violation in vkCreateComputePipelines(), safety serialization before terminating ...\n");
+		bool success = device->serializeToPath(device->getSerializationPath());
+
+		if (success)
+			MessageBoxA(nullptr, "vkCreateComputePipelines() triggered an access violation, the offending state has been serialized.", "CreateComputePipeline Access Violation", 0);
+		else
+			MessageBoxA(nullptr, "vkCreateComputePipelines() triggered an access violation, but the offending state failed to be serialized.", "GraphicsComputePipeline Access Violation", 0);
+		std::terminate();
+	}
+#endif
+}
+
 static VKAPI_ATTR VkResult VKAPI_CALL CreateGraphicsPipelines(VkDevice device, VkPipelineCache pipelineCache,
                                                               uint32_t createInfoCount,
                                                               const VkGraphicsPipelineCreateInfo *pCreateInfos,
@@ -125,8 +190,8 @@ static VKAPI_ATTR VkResult VKAPI_CALL CreateGraphicsPipelines(VkDevice device, V
 		if (layer->isParanoid())
 			layer->serializeToPath(layer->getSerializationPath());
 
-		auto res = layer->getTable()->CreateGraphicsPipelines(device, pipelineCache, 1, &pCreateInfos[i],
-		                                                      pAllocator, &pPipelines[i]);
+		auto res = createGraphicsPipeline(layer, pipelineCache, &pCreateInfos[i], pAllocator, &pPipelines[i]);
+
 		if (res != VK_SUCCESS)
 		{
 			LOGE("Failed to create graphics pipeline, safety serialization ...\n");
@@ -171,8 +236,8 @@ static VKAPI_ATTR VkResult VKAPI_CALL CreateComputePipelines(VkDevice device, Vk
 
 		pPipelines[i] = VK_NULL_HANDLE;
 
-		auto res = layer->getTable()->CreateComputePipelines(device, pipelineCache, 1, &pCreateInfos[i],
-		                                                     pAllocator, &pPipelines[i]);
+		auto res = createComputePipeline(layer, pipelineCache, &pCreateInfos[i], pAllocator, &pPipelines[i]);
+
 		if (res != VK_SUCCESS)
 		{
 			LOGE("Failed to create compute pipeline, safety serialization ...\n");

layer/utils.hpp

@@ -28,6 +28,23 @@
 #include <android/log.h>
 #define LOGI(...) __android_log_print(ANDROID_LOG_INFO, "Fossilize", __VA_ARGS__)
 #define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, "Fossilize", __VA_ARGS__)
+#elif _WIN32
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#define LOGI(...) do { \
+	fprintf(stderr, "Fossilize INFO: " __VA_ARGS__); \
+	fflush(stderr); \
+	char debugString[1024]; \
+	sprintf(debugString, "Fossilize INFO: " __VA_ARGS__); \
+	OutputDebugStringA(debugString); \
+} while(0)
+#define LOGE(...) do { \
+	fprintf(stderr, "Fossilize ERROR: " __VA_ARGS__); \
+	fflush(stderr); \
+	char debugString[1024]; \
+	sprintf(debugString, "Fossilize ERROR: " __VA_ARGS__); \
+	OutputDebugStringA(debugString); \
+} while(0)
 #else
 #include <stdio.h>
 #define LOGI(...) fprintf(stderr, "Fossilize INFO: " __VA_ARGS__)