Skip to content
Permalink
Browse files

Fix handling of failed shared data map creation in overlay.

createSharedDataMap can fail to map the shared data used by other
functionality in the overlay dll. In this case pointers including
"sd" remain NULLd. With the recent overlay changes the modified
shared data structures changed, triggering this case when running
different versions. This crashed Mumble on startup due to an
unchecked dereference of the "sd" pointer.

Also when extracting createSharedDataMap during refactoring behavior
was changed. The overlay kept injecting itself even if the mapping
failed. This also addresses this by making createSharedDataMap return
a bool indicating success or failure.
  • Loading branch information...
hacst committed Nov 30, 2013
1 parent 5f98a65 commit ab12d356218d1570482ae17916a7f67b9e84154c
Showing with 12 additions and 8 deletions.
  1. +12 −8 overlay/lib.cpp
@@ -473,7 +473,7 @@ static LRESULT CALLBACK CallWndProc(int nCode, WPARAM wParam, LPARAM lParam) {
extern "C" __declspec(dllexport) void __cdecl RemoveHooks() {
DWORD dwWaitResult = WaitForSingleObject(hHookMutex, 1000L);
if (dwWaitResult == WAIT_OBJECT_0) {
if (sd->bHooked) {
if (sd != NULL && sd->bHooked) {
if (hhookWnd) {
UnhookWindowsHookEx(hhookWnd);
hhookWnd = NULL;
@@ -487,7 +487,7 @@ extern "C" __declspec(dllexport) void __cdecl RemoveHooks() {
extern "C" __declspec(dllexport) void __cdecl InstallHooks() {
DWORD dwWaitResult = WaitForSingleObject(hHookMutex, 1000L);
if (dwWaitResult == WAIT_OBJECT_0) {
if (! sd->bHooked) {
if (sd != NULL && ! sd->bHooked) {
HMODULE hSelf = NULL;
GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (char *) &InstallHooks, &hSelf);
if (hSelf == NULL) {
@@ -509,7 +509,7 @@ extern "C" __declspec(dllexport) unsigned int __cdecl GetOverlayMagicVersion() {
}

static bool dllmainProcAttachCheckProcessIsBlacklisted(char procname[], char *p);
static void createSharedDataMap();
static bool createSharedDataMap();

static void dllmainProcAttach(char *procname) {
Mutex::init();
@@ -544,7 +544,8 @@ static void dllmainProcAttach(char *procname) {
return;
}

createSharedDataMap();
if(!createSharedDataMap())
return;

if (! bMumble) {
// Hook our own LoadLibrary functions so we notice when a new library (like the d3d ones) is loaded.
@@ -681,23 +682,24 @@ static bool dllmainProcAttachCheckProcessIsBlacklisted(char procname[], char *p)
return false;
}

static void createSharedDataMap() {
static bool createSharedDataMap() {
DWORD dwSharedSize = sizeof(SharedData) + sizeof(Direct3D9Data) + sizeof(DXGIData) + sizeof(D3D10Data);

hMapObject = CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, dwSharedSize, "MumbleOverlayPrivate");
if (hMapObject == NULL) {
ods("Lib: CreateFileMapping failed");
return;
return false;
}

//Note: If the mapping exists dwSharedSize value will be ignored and existing handle returned
bool bInit = (GetLastError() != ERROR_ALREADY_EXISTS);

unsigned char *rawSharedPointer = static_cast<unsigned char *>(
MapViewOfFile(hMapObject, FILE_MAP_ALL_ACCESS, 0, 0, dwSharedSize));

if (rawSharedPointer == NULL) {
ods("Lib: MapViewOfFile Failed");
return;
ods("Lib: MapViewOfFile failed");
return false;
}

if (bInit)
@@ -714,6 +716,8 @@ static void createSharedDataMap() {

d3d10 = reinterpret_cast<D3D10Data *>(rawSharedPointer);
rawSharedPointer += sizeof(D3D10Data);

return true;
}

static void dllmainProcDetach() {

0 comments on commit ab12d35

Please sign in to comment.
You can’t perform that action at this time.