Skip to content
Permalink
Browse files

Merge PR #3160: overlay: Use SIZE_T as correct type for memory addresses

  • Loading branch information...
mkrautz committed Jul 13, 2017
2 parents 49b80d8 + 36fc0f1 commit ea5c03819521bc19542e493d69de23f2b5f71980
Showing with 80 additions and 108 deletions.
  1. +23 −23 overlay/d3d10.cpp
  2. +23 −23 overlay/d3d11.cpp
  3. +11 −30 overlay/d3d9.cpp
  4. +7 −7 overlay/dxgi.cpp
  5. +6 −16 overlay/lib.cpp
  6. +10 −9 overlay/lib.h
@@ -600,7 +600,7 @@ void checkDXGI10Hook(bool preonly) {
return;
}

if (d3d10->iOffsetAddRef == 0 || d3d10->iOffsetRelease == 0) {
if (d3d10->offsetAddRef == 0 || d3d10->offsetRelease == 0) {
return;
}

@@ -639,7 +639,7 @@ void hookD3D10(HMODULE hD3D10, bool preonly) {

if (_wcsicmp(d3d10->wcFileName, modulename) == 0) {
unsigned char *raw = (unsigned char *) hD3D10;
HookAddRelease((voidFunc)(raw + d3d10->iOffsetAddRef), (voidFunc)(raw + d3d10->iOffsetRelease));
HookAddRelease((voidFunc)(raw + d3d10->offsetAddRef), (voidFunc)(raw + d3d10->offsetRelease));
} else if (! preonly) {
ods("D3D10: Interface changed, can't rawpatch. Current: %ls ; Previously: %ls", modulename, d3d10->wcFileName);
} else {
@@ -660,8 +660,8 @@ void PrepareDXGI10(IDXGIAdapter1 *pAdapter, bool initializeDXGIData) {
ods("D3D10: Preparing static data for DXGI and D3D10 Injection");

d3d10->wcFileName[0] = 0;
d3d10->iOffsetAddRef = 0;
d3d10->iOffsetRelease = 0;
d3d10->offsetAddRef = 0;
d3d10->offsetRelease = 0;

HMODULE hD3D10 = LoadLibrary("D3D10.DLL");

@@ -716,31 +716,31 @@ void PrepareDXGI10(IDXGIAdapter1 *pAdapter, bool initializeDXGIData) {
void ***vtbl = (void ***) pSwapChain;

void *pPresent = (*vtbl)[8];
int offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pPresent), dxgi->wcFileName, ARRAY_NUM_ELEMENTS(dxgi->wcFileName), "D3D10", "Present");
if (offset >= 0) {
boost::optional<size_t> offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pPresent), dxgi->wcFileName, ARRAY_NUM_ELEMENTS(dxgi->wcFileName), "D3D10", "Present");
if (offset) {
if (initializeDXGIData) {
dxgi->iOffsetPresent = offset;
ods("D3D10: Successfully found Present offset: %ls: %d", dxgi->wcFileName, dxgi->iOffsetPresent);
dxgi->offsetPresent = *offset;
ods("D3D10: Successfully found Present offset: %ls: %d", dxgi->wcFileName, dxgi->offsetPresent);
} else {
if (dxgi->iOffsetPresent == offset) {
ods("D3D10: Successfully verified Present offset: %ls: %d", dxgi->wcFileName, dxgi->iOffsetPresent);
if (dxgi->offsetPresent == *offset) {
ods("D3D10: Successfully verified Present offset: %ls: %d", dxgi->wcFileName, dxgi->offsetPresent);
} else {
ods("D3D10: Failed to verify Present offset for %ls. Found %d, but previously found %d.", dxgi->wcFileName, offset, dxgi->iOffsetPresent);
ods("D3D10: Failed to verify Present offset for %ls. Found %d, but previously found %d.", dxgi->wcFileName, offset, dxgi->offsetPresent);
}
}
}

void *pResize = (*vtbl)[13];
offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pResize), dxgi->wcFileName, ARRAY_NUM_ELEMENTS(dxgi->wcFileName), "D3D10", "ResizeBuffers");
if (offset >= 0) {
if (offset) {
if (initializeDXGIData) {
dxgi->iOffsetResize = offset;
ods("D3D10: Successfully found ResizeBuffers offset: %ls: %d", dxgi->wcFileName, dxgi->iOffsetResize);
dxgi->offsetResize = *offset;
ods("D3D10: Successfully found ResizeBuffers offset: %ls: %d", dxgi->wcFileName, dxgi->offsetResize);
} else {
if (dxgi->iOffsetResize == offset) {
ods("D3D10: Successfully verified ResizeBuffers offset: %ls: %d", dxgi->wcFileName, dxgi->iOffsetResize);
if (dxgi->offsetResize == *offset) {
ods("D3D10: Successfully verified ResizeBuffers offset: %ls: %d", dxgi->wcFileName, dxgi->offsetResize);
} else {
ods("D3D10: Failed to verify ResizeBuffers offset for %ls. Found %d, but previously found %d.", dxgi->wcFileName, offset, dxgi->iOffsetResize);
ods("D3D10: Failed to verify ResizeBuffers offset for %ls. Found %d, but previously found %d.", dxgi->wcFileName, offset, dxgi->offsetResize);
}
}
}
@@ -749,16 +749,16 @@ void PrepareDXGI10(IDXGIAdapter1 *pAdapter, bool initializeDXGIData) {

void *pAddRef = (*vtbl)[1];
offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pAddRef), d3d10->wcFileName, ARRAY_NUM_ELEMENTS(d3d10->wcFileName), "D3D10", "AddRef");
if (offset >= 0) {
d3d10->iOffsetAddRef = offset;
ods("D3D10: Successfully found AddRef offset: %ls: %d", d3d10->wcFileName, d3d10->iOffsetAddRef);
if (offset) {
d3d10->offsetAddRef = *offset;
ods("D3D10: Successfully found AddRef offset: %ls: %d", d3d10->wcFileName, d3d10->offsetAddRef);
}

void *pRelease = (*vtbl)[2];
offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pRelease), d3d10->wcFileName, ARRAY_NUM_ELEMENTS(d3d10->wcFileName), "D3D10", "Release");
if (offset >= 0) {
d3d10->iOffsetRelease = offset;
ods("D3D10: Successfully found Release offset: %ls: %d", d3d10->wcFileName, d3d10->iOffsetRelease);
if (offset) {
d3d10->offsetRelease = *offset;
ods("D3D10: Successfully found Release offset: %ls: %d", d3d10->wcFileName, d3d10->offsetRelease);
}
}

@@ -608,7 +608,7 @@ void checkDXGI11Hook(bool preonly) {
return;
}

if (d3d11->iOffsetAddRef == 0 || d3d11->iOffsetRelease == 0) {
if (d3d11->offsetAddRef == 0 || d3d11->offsetRelease == 0) {
return;
}

@@ -649,7 +649,7 @@ void hookD3D11(HMODULE hD3D11, bool preonly) {

if (_wcsicmp(d3d11->wcFileName, modulename) == 0) {
unsigned char *raw = (unsigned char *) hD3D11;
HookAddRelease((voidFunc)(raw + d3d11->iOffsetAddRef), (voidFunc)(raw + d3d11->iOffsetRelease));
HookAddRelease((voidFunc)(raw + d3d11->offsetAddRef), (voidFunc)(raw + d3d11->offsetRelease));
} else if (! preonly) {
ods("D3D11: Interface changed, can't rawpatch. Current: %ls ; Previously: %ls", modulename, d3d11->wcFileName);
} else {
@@ -670,8 +670,8 @@ void PrepareDXGI11(IDXGIAdapter1* pAdapter, bool initializeDXGIData) {
ods("D3D11: Preparing static data for DXGI and D3D11 Injection");

d3d11->wcFileName[0] = 0;
d3d11->iOffsetAddRef = 0;
d3d11->iOffsetRelease = 0;
d3d11->offsetAddRef = 0;
d3d11->offsetRelease = 0;

HMODULE hD3D11 = LoadLibrary("D3D11.DLL");

@@ -729,31 +729,31 @@ void PrepareDXGI11(IDXGIAdapter1* pAdapter, bool initializeDXGIData) {
void ***vtbl = (void ***) pSwapChain;

void *pPresent = (*vtbl)[8];
int offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pPresent), dxgi->wcFileName, ARRAY_NUM_ELEMENTS(dxgi->wcFileName), "D3D11", "Present");
if (offset >= 0) {
boost::optional<size_t> offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pPresent), dxgi->wcFileName, ARRAY_NUM_ELEMENTS(dxgi->wcFileName), "D3D11", "Present");
if (offset) {
if (initializeDXGIData) {
dxgi->iOffsetPresent = offset;
ods("D3D11: Successfully found Present offset: %ls: %d", dxgi->wcFileName, dxgi->iOffsetPresent);
dxgi->offsetPresent = *offset;
ods("D3D11: Successfully found Present offset: %ls: %d", dxgi->wcFileName, dxgi->offsetPresent);
} else {
if (dxgi->iOffsetPresent == offset) {
ods("D3D11: Successfully verified Present offset: %ls: %d", dxgi->wcFileName, dxgi->iOffsetPresent);
if (dxgi->offsetPresent == *offset) {
ods("D3D11: Successfully verified Present offset: %ls: %d", dxgi->wcFileName, dxgi->offsetPresent);
} else {
ods("D3D11: Failed to verify Present offset for %ls. Found %d, but previously found %d.", dxgi->wcFileName, offset, dxgi->iOffsetPresent);
ods("D3D11: Failed to verify Present offset for %ls. Found %d, but previously found %d.", dxgi->wcFileName, offset, dxgi->offsetPresent);
}
}
}

void *pResize = (*vtbl)[13];
offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pResize), dxgi->wcFileName, ARRAY_NUM_ELEMENTS(dxgi->wcFileName), "D3D11", "ResizeBuffers");
if (offset >= 0) {
if (offset) {
if (initializeDXGIData) {
dxgi->iOffsetResize = offset;
ods("D3D11: Successfully found ResizeBuffers offset: %ls: %d", dxgi->wcFileName, dxgi->iOffsetResize);
dxgi->offsetResize = *offset;
ods("D3D11: Successfully found ResizeBuffers offset: %ls: %d", dxgi->wcFileName, dxgi->offsetResize);
} else {
if (dxgi->iOffsetResize == offset) {
ods("D3D11: Successfully verified ResizeBuffers offset: %ls: %d", dxgi->wcFileName, dxgi->iOffsetResize);
if (dxgi->offsetResize == *offset) {
ods("D3D11: Successfully verified ResizeBuffers offset: %ls: %d", dxgi->wcFileName, dxgi->offsetResize);
} else {
ods("D3D11: Failed to verify ResizeBuffers offset for %ls. Found %d, but previously found %d.", dxgi->wcFileName, offset, dxgi->iOffsetResize);
ods("D3D11: Failed to verify ResizeBuffers offset for %ls. Found %d, but previously found %d.", dxgi->wcFileName, offset, dxgi->offsetResize);
}
}
}
@@ -762,16 +762,16 @@ void PrepareDXGI11(IDXGIAdapter1* pAdapter, bool initializeDXGIData) {

void *pAddRef = (*vtbl)[1];
offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pAddRef), d3d11->wcFileName, ARRAY_NUM_ELEMENTS(d3d11->wcFileName), "D3D11", "AddRef");
if (offset >= 0) {
d3d11->iOffsetAddRef = offset;
ods("D3D11: Successfully found AddRef offset: %ls: %d", d3d11->wcFileName, d3d11->iOffsetAddRef);
if (offset) {
d3d11->offsetAddRef = *offset;
ods("D3D11: Successfully found AddRef offset: %ls: %d", d3d11->wcFileName, d3d11->offsetAddRef);
}

void *pRelease = (*vtbl)[2];
offset = GetFnOffsetInModule(reinterpret_cast<voidFunc>(pRelease), d3d11->wcFileName, ARRAY_NUM_ELEMENTS(d3d11->wcFileName), "D3D11", "Release");
if (offset >= 0) {
d3d11->iOffsetRelease = offset;
ods("D3D11: Successfully found Release offset: %ls: %d", d3d11->wcFileName, d3d11->iOffsetRelease);
if (offset) {
d3d11->offsetRelease = *offset;
ods("D3D11: Successfully found Release offset: %ls: %d", d3d11->wcFileName, d3d11->offsetRelease);
}
}

@@ -7,8 +7,6 @@
#include <d3d9.h>
#include <time.h>

#undef max // for std::numeric_limits<T>::max()

Direct3D9Data *d3dd = NULL;

typedef IDirect3D9* (WINAPI *pDirect3DCreate9)(UINT SDKVersion) ;
@@ -1011,9 +1009,9 @@ static void hookD3D9(HMODULE hD3D, bool preonly) {
// The module seems to match the one we prepared d3dd for.

unsigned char *raw = (unsigned char *) hD3D;
HookCreateRaw((voidFunc)(raw + d3dd->iOffsetCreate));
if (d3dd->iOffsetCreateEx) {
HookCreateRawEx((voidFunc)(raw + d3dd->iOffsetCreateEx));
HookCreateRaw((voidFunc)(raw + d3dd->offsetCreate));
if (d3dd->offsetCreateEx) {
HookCreateRawEx((voidFunc)(raw + d3dd->offsetCreateEx));
}

} else if (! preonly) {
@@ -1090,19 +1088,10 @@ extern "C" __declspec(dllexport) void __cdecl PrepareD3D9() {
if (!IsFnInModule(reinterpret_cast<voidFunc>(pCreate), d3dd->wcFileName, "D3D9", "CreateDevice")) {
ods("D3D9: CreateDevice is not in D3D9 library");
} else {
unsigned char *fn = reinterpret_cast<unsigned char *>(pCreate);
unsigned char *base = reinterpret_cast<unsigned char *>(hD3D);
unsigned long off = static_cast<unsigned long>(fn - base);

// XXX: convert the offset to use something other than int.
// Issue mumble-voip/mumble#1924.
if (off > static_cast<unsigned long>(std::numeric_limits<int>::max())) {
ods("D3D9: Internal overlay error: CreateDevice offset is > 2GB, does not fit the current data structure.");
} else {
d3dd->iOffsetCreate = static_cast<int>(off);
ods("D3D9: Successfully found prepatch offset: %p %p %p: %d", hD3D, d3dcreate9, pCreate, d3dd->iOffsetCreate);
}
}
size_t fn = reinterpret_cast<size_t>(pCreate);
size_t base = reinterpret_cast<size_t>(hD3D);
d3dd->offsetCreate = fn - base;
ods("D3D9: Successfully found prepatch offset: %p %p %p: %d", hD3D, d3dcreate9, pCreate, d3dd->offsetCreate); }
id3d9->Release();
}
}
@@ -1125,18 +1114,10 @@ extern "C" __declspec(dllexport) void __cdecl PrepareD3D9() {
if (!IsFnInModule(reinterpret_cast<voidFunc>(pCreateEx), d3dd->wcFileName, "D3D9", "CreateDeviceEx")) {
ods("D3D9: CreateDeviceEx is not in D3D9 library");
} else {
unsigned char *fn = reinterpret_cast<unsigned char *>(pCreateEx);
unsigned char *base = reinterpret_cast<unsigned char *>(hD3D);
unsigned long off = static_cast<unsigned long>(fn - base);

// XXX: convert the offset to use something other than int.
// Issue mumble-voip/mumble#1924.
if (off > static_cast<unsigned long>(std::numeric_limits<int>::max())) {
ods("D3D9: Internal overlay error: CreateDeviceEx offset is > 2GB, does not fit the current data structure.");
} else {
d3dd->iOffsetCreateEx = static_cast<int>(off);
ods("D3D9: Successfully found prepatch ex offset: %p %p %p: %d", hD3D, d3dcreate9ex, pCreateEx, d3dd->iOffsetCreateEx);
}
size_t fn = reinterpret_cast<size_t>(pCreateEx);
size_t base = reinterpret_cast<size_t>(hD3D);
d3dd->offsetCreateEx = fn - base;
ods("D3D9: Successfully found prepatch ex offset: %p %p %p: %d", hD3D, d3dcreate9ex, pCreateEx, d3dd->offsetCreateEx);
}

id3d9->Release();
@@ -120,7 +120,7 @@ void checkDXGIHook(bool preonly) {
return;
}

if (dxgi->iOffsetPresent == 0 || dxgi->iOffsetResize == 0)
if (dxgi->offsetPresent == 0 || dxgi->offsetResize == 0)
return;

bCheckHookActive = true;
@@ -158,8 +158,8 @@ void hookDXGI(HMODULE hDXGI, bool preonly) {
// The module seems to match the one we prepared d3dd for.

unsigned char *raw = (unsigned char *) hDXGI;
HookPresentRaw((voidFunc)(raw + dxgi->iOffsetPresent));
HookResizeRaw((voidFunc)(raw + dxgi->iOffsetResize));
HookPresentRaw((voidFunc)(raw + dxgi->offsetPresent));
HookResizeRaw((voidFunc)(raw + dxgi->offsetResize));

} else if (! preonly) {
ods("DXGI: Interface changed, can't rawpatch. Current: %ls ; Previously: %ls", modulename, dxgi->wcFileName);
@@ -184,8 +184,8 @@ extern "C" __declspec(dllexport) void __cdecl PrepareDXGI() {
ods("DXGI: Preparing static data for DXGI Injection");

dxgi->wcFileName[0] = 0;
dxgi->iOffsetPresent = 0;
dxgi->iOffsetResize = 0;
dxgi->offsetPresent = 0;
dxgi->offsetResize = 0;

// Make sure this is Vista or greater as quite a number of <=WinXP users have fake DX10 libs installed
OSVERSIONINFOEXW ovi;
@@ -214,9 +214,9 @@ extern "C" __declspec(dllexport) void __cdecl PrepareDXGI() {
pFactory->EnumAdapters1(0, &pAdapter);

/// Offsets have to be identified and initialized only once.
bool initializeDXGIData = !dxgi->iOffsetPresent && !dxgi->iOffsetResize;
bool initializeDXGIData = !dxgi->offsetPresent && !dxgi->offsetResize;
PrepareDXGI10(pAdapter, initializeDXGIData);
initializeDXGIData = !dxgi->iOffsetPresent && !dxgi->iOffsetResize;
initializeDXGIData = !dxgi->offsetPresent && !dxgi->offsetResize;
PrepareDXGI11(pAdapter, initializeDXGIData);

pFactory->Release();
@@ -8,8 +8,6 @@

#include "overlay_exe/overlay_exe.h"

#undef max // for std::numeric_limits<T>::max()

static HANDLE hMapObject = NULL;
static HANDLE hHookMutex = NULL;
static HHOOK hhookWnd = 0;
@@ -706,13 +704,13 @@ bool IsFnInModule(voidFunc fnptr, wchar_t *refmodulepath, const std::string &log
return false;
}

int GetFnOffsetInModule(voidFunc fnptr, wchar_t *refmodulepath, unsigned int refmodulepathLen, const std::string &logPrefix, const std::string &fnName) {
boost::optional<size_t> GetFnOffsetInModule(voidFunc fnptr, wchar_t *refmodulepath, unsigned int refmodulepathLen, const std::string &logPrefix, const std::string &fnName) {

HMODULE hModule = NULL;

if (! GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast<LPCTSTR>(fnptr), &hModule)) {
ods((logPrefix + ": Failed to get module for " + fnName).c_str());
return -1;
return boost::none;
}

const bool bInit = refmodulepath[0] == '\0';
@@ -723,20 +721,12 @@ int GetFnOffsetInModule(voidFunc fnptr, wchar_t *refmodulepath, unsigned int ref
GetModuleFileNameW(hModule, modulename, ARRAY_NUM_ELEMENTS(modulename));
if (_wcsicmp(modulename, refmodulepath) != 0) {
ods((logPrefix + ": " + fnName + " functions module path does not match previously found. Now: '%ls', Previously: '%ls'").c_str(), modulename, refmodulepath);
return -2;
return boost::none;
}
}

unsigned char *fn = reinterpret_cast<unsigned char *>(fnptr);
unsigned char *base = reinterpret_cast<unsigned char *>(hModule);
unsigned long off = static_cast<unsigned long>(fn - base);

// XXX: convert this function to use something other than int.
// Issue mumble-voip/mumble#1924.
if (off > static_cast<unsigned long>(std::numeric_limits<int>::max())) {
ods("Internal overlay error: GetFnOffsetInModule() offset greater than return type can hold.");
return -1;
}
size_t fn = reinterpret_cast<size_t>(fnptr);
size_t base = reinterpret_cast<size_t>(hModule);

return static_cast<int>(off);
return fn - base;
}

0 comments on commit ea5c038

Please sign in to comment.
You can’t perform that action at this time.