Improve vk_debug output a lot by throwing away the useless parts of t…

…he message and limit the callstack to the first 5 gzdoom calls

src/common/rendering/vulkan/system/vk_device.cpp

@@ -47,7 +47,7 @@
 bool I_GetVulkanPlatformExtensions(unsigned int *count, const char **names);
 bool I_CreateVulkanSurface(VkInstance instance, VkSurfaceKHR *surface);
 
-FString JitCaptureStackTrace(int framesToSkip, bool includeNativeFrames);
+FString JitCaptureStackTrace(int framesToSkip, bool includeNativeFrames, int maxFrames = -1);
 
 // Physical device info
 static std::vector<VulkanPhysicalDevice> AvailableDevices;
@@ -404,14 +404,25 @@ VkBool32 VulkanDevice::DebugCallback(VkDebugUtilsMessageSeverityFlagBitsEXT mess
 
 	FString msg = callbackData->pMessage;
 
-	// For patent-pending reasons the validation layer apparently can't do this itself..
-	for (uint32_t i = 0; i < callbackData->objectCount; i++)
+	// Attempt to parse the string because the default formatting is totally unreadable and half of what it writes is totally useless!
+	auto parts = msg.Split("|");
+	if (parts.Size() == 3)
 	{
-		if (callbackData->pObjects[i].pObjectName)
+		msg = parts[2];
+		auto pos = msg.IndexOf(" The Vulkan spec states:");
+		if (pos >= 0)
+			msg = msg.Left(pos);
+
+		if (callbackData->objectCount > 0)
 		{
-			FString hexname;
-			hexname.Format("0x%" PRIx64, callbackData->pObjects[i].objectHandle);
-			msg.Substitute(hexname.GetChars(), callbackData->pObjects[i].pObjectName);
+			msg += " (";
+			for (uint32_t i = 0; i < callbackData->objectCount; i++)
+			{
+				if (i > 0)
+					msg += ", ";
+				msg += callbackData->pObjects[i].pObjectName;
+			}
+			msg += ")";
 		}
 	}
 
@@ -454,7 +465,7 @@ VkBool32 VulkanDevice::DebugCallback(VkDebugUtilsMessageSeverityFlagBitsEXT mess
 
 			if (vk_debug_callstack && showcallstack)
 			{
-				FString callstack = JitCaptureStackTrace(0, true);
+				FString callstack = JitCaptureStackTrace(0, true, 5);
 				if (!callstack.IsEmpty())
 					Printf("%s\n", callstack.GetChars());
 			}

src/common/scripting/jit/jit.h

@@ -5,4 +5,4 @@
 
 JitFuncPtr JitCompile(VMScriptFunction *func);
 void JitDumpLog(FILE *file, VMScriptFunction *func);
-FString JitCaptureStackTrace(int framesToSkip, bool includeNativeFrames);
+FString JitCaptureStackTrace(int framesToSkip, bool includeNativeFrames, int maxFrames = -1);

src/common/scripting/jit/jit_runtime.cpp

@@ -5,6 +5,7 @@
 
 #ifdef WIN32
 #include <DbgHelp.h>
+#include <psapi.h>
 #else
 #include <execinfo.h>
 #include <cxxabi.h>
@@ -813,8 +814,16 @@ static int CaptureStackTrace(int max_frames, void **out_frames)
 class NativeSymbolResolver
 {
 public:
-	NativeSymbolResolver() { SymInitialize(GetCurrentProcess(), nullptr, TRUE); }
-	~NativeSymbolResolver() { SymCleanup(GetCurrentProcess()); }
+	NativeSymbolResolver()
+	{
+		SymInitialize(GetCurrentProcess(), nullptr, TRUE);
+		GetModuleInformation(GetCurrentProcess(), GetModuleHandle(0), &moduleInfo, sizeof(MODULEINFO));
+	}
+
+	~NativeSymbolResolver()
+	{
+		SymCleanup(GetCurrentProcess());
+	}
 
 	FString GetName(void *frame)
 	{
@@ -830,6 +839,9 @@ class NativeSymbolResolver
 		BOOL result = SymGetSymFromAddr64(GetCurrentProcess(), (DWORD64)frame, &displacement, symbol64);
 		if (result)
 		{
+			if ((DWORD64)frame < (DWORD64)moduleInfo.lpBaseOfDll || (DWORD64)frame >= ((DWORD64)moduleInfo.lpBaseOfDll + moduleInfo.SizeOfImage))
+				return s; // Ignore anything not from the exe itself
+
 			IMAGEHLP_LINE64 line64;
 			DWORD displacement1 = 0;
 			memset(&line64, 0, sizeof(IMAGEHLP_LINE64));
@@ -847,6 +859,8 @@ class NativeSymbolResolver
 
 		return s;
 	}
+
+	MODULEINFO moduleInfo = {};
 };
 #else
 class NativeSymbolResolver
@@ -952,7 +966,7 @@ FString JitGetStackFrameName(NativeSymbolResolver *nativeSymbols, void *pc)
 	return nativeSymbols ? nativeSymbols->GetName(pc) : FString();
 }
 
-FString JitCaptureStackTrace(int framesToSkip, bool includeNativeFrames)
+FString JitCaptureStackTrace(int framesToSkip, bool includeNativeFrames, int maxFrames)
 {
 	void *frames[32];
 	int numframes = CaptureStackTrace(32, frames);
@@ -961,10 +975,18 @@ FString JitCaptureStackTrace(int framesToSkip, bool includeNativeFrames)
 	if (includeNativeFrames)
 		nativeSymbols.reset(new NativeSymbolResolver());
 
+	int total = 0;
 	FString s;
 	for (int i = framesToSkip + 1; i < numframes; i++)
 	{
-		s += JitGetStackFrameName(nativeSymbols.get(), frames[i]);
+		FString name = JitGetStackFrameName(nativeSymbols.get(), frames[i]);
+		if (!name.IsEmpty())
+		{
+			s += name;
+			total++;
+			if (maxFrames != -1 && maxFrames == total)
+				break;
+		}
 	}
 	return s;
 }