DHooks: Error on argument passflags for detours (#1773)

The passflags are only supported by SourceHook for virtual hooks and are ignored for detours with DynamicDetours. This caused confusion, so throw an error when trying to set e.g. the DHookPass_ByRef flag on detour arguments.

extensions/dhooks/natives.cpp

@@ -256,6 +256,15 @@ cell_t Native_SetFromConf(IPluginContext *pContext, const cell_t *params)
 	setup->funcAddr = addr;
 	setup->offset = offset;
 
+	if (addr == nullptr)
+	{
+		setup->hookMethod = Virtual;
+	}
+	else
+	{
+		setup->hookMethod = Detour;
+	}
+
 	return 1;
 }
 
@@ -282,6 +291,13 @@ cell_t Native_AddParam(IPluginContext *pContext, const cell_t *params)
 		info.flags = PASSFLAG_BYVAL;
 	}
 
+	// DynamicDetours doesn't expose the passflags concept like SourceHook.
+	// See if we're trying to set some invalid flags on detour arguments.
+	if(setup->hookMethod == Detour && (info.flags & ~PASSFLAG_BYVAL) > 0)
+	{
+		return pContext->ThrowNativeError("Pass flags are only supported for virtual hooks.");
+	}
+
 	if (params[0] >= 5)
 	{
 		PluginRegister custom_register = (PluginRegister)params[5];

extensions/dhooks/signatures.cpp

@@ -376,6 +376,20 @@ SMCResult SignatureGameConfig::ReadSMC_LeavingSection(const SMCStates *states)
 			return SMCResult_HaltFail;
 		}
 
+		if (!g_CurrentSignature->offset.length())
+		{
+			// DynamicDetours doesn't expose the passflags concept like SourceHook.
+			// See if we're trying to set some invalid flags on detour arguments.
+			for (auto &arg : g_CurrentSignature->args)
+			{
+				if ((arg.info.flags & ~PASSFLAG_BYVAL) > 0)
+				{
+					smutils->LogError(myself, "Function \"%s\" uses unsupported pass flags in argument \"%s\". Flags are only supported for virtual hooks: line: %i col: %i", g_CurrentFunctionName.c_str(), arg.name.c_str(), states->line, states->col);
+					return SMCResult_HaltFail;
+				}
+			}
+		}
+
 		// Save this function signature in our cache.
 		signatures_.insert(g_CurrentFunctionName.c_str(), g_CurrentSignature);
 		g_CurrentFunctionName = "";

extensions/dhooks/vhook.h

@@ -225,6 +225,11 @@ class HookParamsStruct
 	DHooksInfo *dg;
 };
 
+enum HookMethod {
+	Virtual,
+	Detour
+};
+
 class HookSetup
 {
 public:
@@ -238,6 +243,7 @@ class HookSetup
 		this->offset = offset;
 		this->funcAddr = nullptr;
 		this->callback = callback;
+		this->hookMethod = Virtual;
 	};
 	HookSetup(ReturnType returnType, unsigned int returnFlag, CallingConvention callConv, ThisPointerType thisType, void *funcAddr)
 	{
@@ -249,6 +255,7 @@ class HookSetup
 		this->offset = -1;
 		this->funcAddr = funcAddr;
 		this->callback = nullptr;
+		this->hookMethod = Detour;
 	};
 	~HookSetup(){};
 
@@ -266,6 +273,7 @@ class HookSetup
 	int offset;
 	void *funcAddr;
 	IPluginFunction *callback;
+	HookMethod hookMethod;
 };
 
 class DHooksManager